Barista/gui/main_window/docks/properties/property_widgets/__init__.py 0000664 ڊ0011117 00000000000 13265130053 026473 0 ustar klemms o0cvpr Barista/gui/main_window/docks/properties/property_widgets/data.py 0000664 ڊ0011117 00000102175 13265130053 025665 0 ustar klemms o0cvpr from PyQt5.QtCore import QObject, pyqtSignal
import backend.caffe.proto_info as info
from gui.main_window.docks.properties.property_widgets.types import *
class ReasonUpdated:
""" This class represents an update of a value in PropertyDataObject """
def __init__(self, oldValue, newValue):
self.oldValue = oldValue
self.newValue = newValue
def __str__(self):
return "Updated {} to {}".format(self.oldValue, self.newValue)
def __eq__(self, other):
if type(other) != type(self): return False
return other.oldValue == self.oldValue and other.newValue == self.newValue
class ReasonAdded:
""" This class represents a push in a list in PropertyDataListObject """
def __init__(self, idx):
self.idx = idx
def __str__(self):
return "Added {}".format(self.idx)
def __eq__(self, other):
if type(other) != type(self): return False
return other.idx == self.idx
class ReasonRemoved:
""" This class represents a remove in a list in PropertyDataListObject """
def __init__(self, idx, val):
self.idx = idx
self.value = val
def __str__(self):
return "Removed {} - value {}".format(self.idx, self.value)
def __eq__(self, other):
if type(other) != type(self): return False
return other.idx == self.idx
class ReasonGiven:
""" This class represents a new given value in PropertyData """
def __init__(self, name):
self.name = name
def __str__(self):
return "Given {}".format(self.name)
def __eq__(self, other):
if type(other) != type(self): return False
return other.name == self.name
class ReasonUngiven:
""" This class represents a value in PropertyData which became deleted """
def __init__(self, name):
self.name = name
def __str__(self):
return "Ungiven {}".format(self.name)
def __eq__(self, other):
if type(other) != type(self): return False
return other.name == self.name
class ReasonKeyAdded:
""" This class represents a key/value in PropertyDataDictObject which was added """
def __init__(self, key):
self.key = key
def __str__(self):
return "Key added {}".format(self.key)
def __eq__(self, other):
if type(other) != type(self): return False
return other.key == self.key
class ReasonKeyDeleted:
""" This class represents a key/value in PropertyDataDictObject which was deleted """
def __init__(self, key):
self.key = key
def __str__(self):
return "Key deleted {}".format(self.key)
def __eq__(self, other):
if type(other) != type(self): return False
return other.key == self.key
class PropertyInfoBuilder:
def buildInfo(self, name, protoparameter, uri=None):
""" This function should build an info object
(something which behaves like PropertyInfo)
for the given parameter.
name is the name of the info object
protoparameter is any object which helps build the info object
uri is a list whicht describes the position of this info
"""
pass
class PropertyInfo(object):
""" This Class gives infos about a type in properties """
def typeEntryString(self):
""" Give the type as String.
If the type is a list/dictionary, returns the type of the entries.
Only needed if entryInfo won't be overwritten
"""
pass
def entryInfo(self):
""" Returns a info object which describes a entry object
if the type is a list/dictionary
"""
return PropertyInfoListEntryWrapper(self)
def typeString(self):
""" Give the type as String (see StringType, IntType,...)"""
pass
def enumOptions(self):
""" A list of options if enum """
pass
def name(self):
""" Return the name of this property """
pass
def deprecated(self):
""" Return true iff this property is deprecated """
pass
def description(self):
""" Return a decription of this property """
pass
def prototype(self):
""" Return a raw value of a default type.
The result may be changed in other function,
so you may want to return a copy of your prototype
"""
pass
def subparameter(self):
""" Should return a dictionary which keys are the subparameter
and its values are the protoparameter for your PropertyInfoBuilder
and can be from any type.
E.g. {"subparameter1": SomeCaffeMetaInfosForExampleIfInfoBuilderWantsThis,
"subparameter2": MaySomeVeryDifferentData
}
"""
pass
def isEditable(self):
pass
def isRequired(self):
pass
class PropertyInfoListEntryWrapper(PropertyInfo):
""" Wraps info such that typeString returns the typeEntryString.
Useful for dictionary or list subtypes
"""
def __init__(self, info):
self._info = info #type: PropertyInfo
def base(self):
""" Return the base info-object this object wraps """
return self._info
def typeString(self):
return self._info.typeEntryString()
def typeEntryString(self):
return self._info.typeEntryString()
def entryInfo(self):
raise LookupError("EntryInfo should not be used twice by using PropertyInfoListEntryWrapper")
def enumOptions(self):
return self._info.enumOptions()
def name(self):
return self._info.name()
def deprecated(self):
return self._info.deprecated()
def description(self):
return self._info.description()
def prototype(self):
return self._info.prototype()
def isEditable(self):
return self._info.isEditable()
def isRequired(self):
return self._info.isRequired()
def subparameter(self):
return self._info.subparameter()
def propertyObjectForInfo(info):
""" Returns the subtype of PropertyDataObject
which should be used with the given type
in info
"""
if info.typeString() == GroupType:
return PropertyDataGroupObject
if info.typeString() == ListType:
return PropertyDataListObject
if info.typeString() == DictType:
return PropertyDataDictObject
return PropertyDataObject
def dataBuild(parentContainer, idx, info, infobuilder, subtype=None, parent=None, uri=[], signalcontroller = None):
""" Builds the value from the raw value given by the parent container
(e.g. Dictionary or List) and idx.
If raw value is primitive, returns a primitive.
If value is a list, returns a list of PropertyData/-Object.
If value is Group-Type, returns a PropertyData.
info is from type PropertyInfo and contains the information
of parentContainer[idx].
infobuilder: instance of PropertyInfoBuilder for
generating meta informations
parent is a PropertyDataObject which someChildPropertyChanged will be connected with its childs.
uri is a identifier for this entry in the whole dictionary (type: list)
"""
# Here we build Data for an entry of an container (e.g. list, dict).
# Without subtype=="property" an primitive can be returned, but here
# this primitive is capsuled in a PropertyDataObject
if subtype == "Property":
PData = propertyObjectForInfo(info)
res = PData(info, parentContainer, idx, infobuilder, uri=uri, signalcontroller=signalcontroller)
if parent:
res.someChildPropertyChanged.connect(parent.someChildPropertyChanged)
return res
# Group-Type -> Build PropertyData and connect change-signal-propagation
if info.typeString() == GroupType:
res = PropertyData(parentContainer[idx],info.subparameter(),infobuilder, uri=uri, signalcontroller=signalcontroller)
if parent:
res.someChildPropertyChanged.connect(parent.someChildPropertyChanged)
return res
# List -> Recursive build data for every entry
# and connect change-signal-propagation for each
if info.typeString() == ListType:
res = []
for i in range(len(parentContainer[idx])):
data = dataBuild(parentContainer[idx], i, info.entryInfo(), infobuilder, "Property", uri=uri+[i], signalcontroller=signalcontroller)
if parent:
data.someChildPropertyChanged.connect(parent.someChildPropertyChanged)
res.append(data)
return res
# Dictionary -> Recursive build data for every key
# and connect change-signal-propagation for each
if info.typeString() == DictType:
res = {}
me = parentContainer[idx]
for key in me:
data = dataBuild(me, key, info.entryInfo(), infobuilder, "Property", uri=uri+[key], signalcontroller=signalcontroller)
if parent:
data.someChildPropertyChanged.connect(parent.someChildPropertyChanged)
res[key] = data
return res
return parentContainer[idx]
import traceback
import sys
class SignalController(object):
""" Class which can delay signals """
def __init__(self):
self.__locked = 0 # Counter how much lock-calls are present
self._cache = [] # Cache every Signal, which is not emitted yet
self.__loging = False # Debugging -> Logging to console
def enableLoging(self):
""" Let this object print a stack-trace on every emitLater call
(Debugging purpose)
"""
self.__loging = True
def disableLoging(self):
""" Stop this object from printing a stack-trace on every emitLater call
(Debugging purpose)
"""
self.__loging = False
def isLocked(self):
return self.__locked > 0
def emitLater(self, signal, *args):
""" emit the signal if not locked, otherwhise wait until unlock """
if self.__loging:
print("Signal log - Traceback")
for item in traceback.extract_stack():
print(item)
fun = lambda: signal.emit(*args)
if self.isLocked():
self._cache.append(fun)
else:
fun()
def lock(self):
""" lock the controller, such that the signals get emit when it's unlocked """
self.__locked += 1
def unlock(self):
""" unlock the controller and emit all cached signals """
if self.__locked == 0:
return
self.__locked -= 1
if not self.isLocked():
tmp = self._cache
self._cache = []
for fun in tmp:
fun()
def lockWhileFunction(self,fun):
""" Lock this object - then call fun - then unlock this object.
It can happened that the developer forgot to unlock
for example when the function fun returns at some point.
It is saver to use the function with lockWhileFunction
"""
self.lock()
res = fun()
self.unlock()
return res
class PropertyDataObject(QObject):
""" Represent a property with a value """
propertyChanged = pyqtSignal(object, object) # second argument is reason for changing
someChildPropertyChanged = pyqtSignal(list, object)
def __init__(self, info, parentcontainer, idx, infobuilder, uri=[], parent=None, signalcontroller=None):
""" info: PropertyInfo like object,
parentdict: the dictionary/list containing this value
idx: the idx, can be integer or string
uri: a identifier for this property
e.g. uri = ['exclude', 2, 'phase']
infobuilder: instance of PropertyInfoBuilder
singalController: instance of SignalController for delay signals
"""
super(PropertyDataObject, self).__init__(parent)
self._uri = uri
self._info = info #type: PropertyInfo
self._parentcontainer = parentcontainer #type: list
self._idx = idx
self._propvalue = None
self.propertyChanged.connect(lambda _, reason: self.someChildPropertyChanged.emit(self._uri, reason))
self._infobuilder = infobuilder
self._signalcontroller = signalcontroller # type: SignalController
def signalController(self):
""" Returns an object which can delay signals.
It will delay signals for all docks in this hierarchy.
That means that the child and parents DataObjects share this signalController
and locking it will cause every object here to delay its signals.
"""
return self._signalcontroller
def uri(self):
return self._uri
def _setRawValue(self, val):
""" Set the value in the wrapped dictionary """
self._parentcontainer[self._idx] = val
def _rawValue(self):
""" Get the value from the wrapped dictionary """
return self._parentcontainer[self._idx]
def _buildValue(self):
""" Build the suitable PropertyData.. for the represented value """
res = dataBuild(self._parentcontainer, self._idx, self._info,self._infobuilder, parent=self, uri=self.uri(), signalcontroller=self._signalcontroller)
return res
def info(self):
"""Should return an PropertyInfo like Object"""
return self._info
def value(self):
""" Return the value. If it is primitiv, the primitiv.
If it is a group returns a PropertyData.
If it is repeated returns a list of the PropertyDataObject
"""
if self._propvalue is None:
self._propvalue = self._buildValue()
return self._propvalue
def listGetValue(self):
""" Return the value, which should be returned on ___getitem___ of the parent"""
return self.value()
def setValue(self, value):
""" Update the value of this PropertyDataObject.
value should be a raw type not something like another PropertyDataObject.
If the value has a wrong type an exception get raised.
E.g. propertydata.setValue(42)
propertydata.setValue(["a","b"])
"""
if not checkIsType(value, self.info().typeString()):
raise ValueError("Expected {} found type {} at {}".format(self.info().typeString(), type(value),self._uri))
oldVal = self._rawValue()
if oldVal == value:
return
self._setRawValue(value)
self._propvalue = None
self._emitChange(value, ReasonUpdated(oldVal, value))
def _emitChange(self, value, reason):
""" Emit change signal with support for SignalController """
if self._signalcontroller:
self._signalcontroller.emitLater(self.propertyChanged, value, reason)
else:
self.propertyChanged.emit(value, reason)
def valueOf(self, relative_uri):
""" Return the value of a child element.
relative_uri is the uri (a dictionary with the path like
["prop1", "prop2"])
which is relative to this PropertyDataObject.
E.g. if we are prop1 the relative uri would
look like ["prop2"].
If no such element found None will be returned.
"""
val = self.value()
# We want this object
if relative_uri == []:
return val
# We want a child object
idx = relative_uri[0]
if isinstance(val, PropertyData):
return val.valueOf(relative_uri)
# We have no childs => invalid => return None
return None
def __str__(self):
return "{} at {}".format(self.__class__.__name__,self.uri())
def prototypeOfType(info):
""" Return some default value for the given type in info.
Useful for a entry in a list.
"""
info = info #type: PropertyInfo
entrytype = info.entryInfo().typeString()
# May return info.entryInfo().prototype() a better solution?!
if entrytype == StringType:
return ""
if entrytype == BoolType:
return False
if entrytype == IntType:
return 0
if entrytype == FloatType:
return 0.0
if entrytype == GroupType:
return {}
if entrytype == EnumType:
return info.enumOptions()[0]
if entrytype == UnknownType:
return None
if entrytype == DictType:
return {}
class PropertyDataBuildingException(Exception):
pass
class PropertyDataGroupObject(PropertyDataObject):
""" Represent a PropertyDataObject containing PropertyData
It pass through the getitem, setitem, deltitem, contains operations
"""
def listGetValue(self):
""" Return the value, which should be returned on ___getitem___ of the parent"""
return self.value()
def __getitem__(self, idx):
return self.value()[idx]
def __setitem__(self, idx, val):
self.value()[idx] = val
def __delitem__(self, idx):
del self.value()[idx]
def __contains__(self,key):
return key in self.value()
class PropertyDataDictObject(PropertyDataObject):
""" Represent a PropertyDataObject containing a dictionary"""
class itemiter(object):
def __init__(self, parent):
self.parent = parent # type: PropertyDataDictObject
def __iter__(self):
for key in self.parent:
yield key, self.parent[key]
def iteritems(self):
return PropertyDataDictObject.itemiter(self)
def keys(self):
return self._rawValue().keys()
def __getitem__(self, idx):
return self.value()[idx]
def __setitem__(self, idx, val):
if idx in self:
if self.info().entryInfo().typeString() in [ListType, GroupType]:
# We can not change PropertyData e.g. because
# they are built by a fix procedure
raise ValueError("Can only change primitive types through this function, not list or groups")
self.value()[idx].setValue(val)
else:
self._rawValue()[idx] = val
newdata = dataBuild(self._rawValue(), idx, self.info().entryInfo(), self._infobuilder, "Property", uri=self.uri()+[idx], signalcontroller=self._signalcontroller)
newdata.someChildPropertyChanged.connect(self.someChildPropertyChanged)
self._propvalue[idx] = newdata
#self._propvalue = self._buildValue()
self._emitChange(self.value(), ReasonKeyAdded(idx))
def __delitem__(self, idx):
v=self._rawValue()#type: list
del v[idx]
#self._propvalue = self._buildValue()
del self._propvalue[idx]
self._emitChange(self.value(), ReasonKeyDeleted(idx))
def __contains__(self,key):
return key in self.value()
def __iter__(self):
for key in self.value():
yield key
def getBoringDict(self):
""" Return the underlying dictionary this object operates on """
return self._rawValue()
def listGetValue(self):
""" Return the value, which should be returned on ___getitem___ of the parent"""
return self
def setValue(self, value):
if not checkIsType(value, self.info().typeString()):
raise ValueError("Expected {} found type {} at {}".format(self.info().typeString(), type(value), self._uri))
oldVal = self._rawValue()
if oldVal == value:
return
self._setRawValue(value)
self._propvalue = self._buildValue()
self._emitChange(self.value(), ReasonUpdated(oldVal, value))
def valueOf(self, relative_uri):
val = self.value()
# We want this object
if relative_uri == []:
return val
# We want a child object
idx = relative_uri[0]
# Uri element id not an index or not valid
if type(idx) != str or not idx in self:
return None
# Call the function of child with right relative uri
if isinstance(val[idx], PropertyDataObject) or isinstance(val[idx], PropertyData):
return val[idx].valueOf(relative_uri[1:])
# If the child is a primitive value but the uri want to
# step into the next element => Invalid => return None
return None
class PropertyDataListObject(PropertyDataObject):
""" Represent a property with multiple values """
def listGetValue(self):
""" Return the value, which should be returned on ___getitem___ of the parent"""
return self
def pushBack(self, value):
""" Append the raw value to the list """
idx = len(self._rawValue())
self._rawValue().append(value)
if self._propvalue is None:
self._propvalue = self._buildValue()
else:
newdata = dataBuild(self._rawValue(), idx, self.info().entryInfo(), self._infobuilder, "Property", uri=self.uri()+[idx], signalcontroller=self._signalcontroller)
newdata.someChildPropertyChanged.connect(self.someChildPropertyChanged)
self._propvalue.append(newdata)
#self._propvalue = self._buildValue()
self._emitChange(self.value(), ReasonAdded(len(self._rawValue())-1))
def removeObject(self, idx):
v=self._rawValue()#type: list
oldval = v[idx]
del v[idx]
self._propvalue = self._buildValue()
self._emitChange(self.value(), ReasonRemoved(idx, oldval))
def insert(self, idx, val):
v=self._rawValue()#type: list
v.insert(idx,val)
self._propvalue = self._buildValue()
self._emitChange(self.value(), ReasonAdded(idx))
def remove(self, val):
v=self._rawValue()#type: list
idx = v.index(val)
v.remove(val)
self._propvalue = self._buildValue()
self._emitChange(self.value(), ReasonRemoved(idx, val))
def index(self,val):
""" Returns the index of val """
return self._rawValue().index(val)
def setValue(self, value):
if not checkIsType(value, self.info().typeString()):
raise ValueError("Expected {} found type {} at {}".format(self.info().typeString(), type(value), self._uri))
oldVal = self._rawValue()
if oldVal == value:
return
self._setRawValue(value)
self._propvalue = self._buildValue()
self._emitChange(self.value(), ReasonUpdated(oldVal, value))
def prototypeValue(self):
""" Returns a raw value (not something like PropertyDataObject)
from an new object in this list
"""
return prototypeOfType(self.info())
def valueOf(self, relative_uri):
""" Return the value of a child element.
relative_uri is the uri (a dictionary with the path like
["listprop", 1, "prop2"])
which is relative to this PropertyDataListObject.
E.g. if we are listprop the relative uri would
look like [1, "prop2"].
If no such element found None will be returned.
"""
val = self.value()
# We want this object
if relative_uri == []:
return val
# We want a child object
idx = relative_uri[0]
# Uri element id not an index or not valid
if type(idx) != int or idx < 0 or idx >= len(val):
return None
# Call the function of child with right relative uri
if isinstance(val[idx], PropertyDataObject) or isinstance(val[idx], PropertyData):
return val[idx].valueOf(relative_uri[1:])
# If the child is a primitive value but the uri want to
# step into the next element => Invalid => return None
return None
def __getitem__(self, idx):
item = self.value()[idx]
return item.listGetValue()
def __setitem__(self, idx, val):
item = self.value()[idx]
if item.info().typeString() in [ListType, GroupType]:
# We can not change PropertyData e.g. because
# they are builded by a fix procedure
raise ValueError("Can only change primitive types through this function, not list or groups")
item.setValue(val)
def __delitem__(self, idx):
self.removeObject(idx)
def __iter__(self):
for el in self.value():
yield el.listGetValue()
def __len__(self):
return len(self._rawValue())
def __contains__(self, item):
for el in self:
if el == item:
return True
return False
class PropertyData(QObject):
""" Reresent a collection of properties """
# Gets emit, whenever some property get changed
# Get called with an identifier list which looks like this
# ["propname","propnameoflist",3,"antoherchild","someItem"]
# and a object which represent the reason for emitting
# (e.g. ReasonUpdated, ReasonGiven, ...)
someChildPropertyChanged = pyqtSignal(list, object)
# propertyGotGiven (value: PropertyDataObject)
propertyGotGiven = pyqtSignal(object)
# propertyGotUnGiven (name: str)
propertyGotUnGiven = pyqtSignal(str)
def __init__(self, valuedict, metadict, infobuilder, uri=[],parent=None, signalcontroller = None):
""" valudict: Recursive list of {key: value}
metadict: List of {key: protoinfo.Parameter }
infobuilder: instance of PropertyInfoBuilder
which generates the meta informations used here
"""
super(PropertyData, self).__init__(parent)
if type(valuedict) != dict:
raise PropertyDataBuildingException("valuedict should be a dict, is {} instead at {}".format(type(valuedict), uri))
self.valuedict = valuedict #type: dict
if type(metadict) != dict:
raise PropertyDataBuildingException("Metadict should be a dict, is {} instead at {}".format(type(metadict), uri))
self.metadict = metadict #type: dict
self._buildProps = None
self._uri = uri
self._infobuilder = infobuilder #type: PropertyInfoBuilder
self._signalcontroller = signalcontroller #type: SignalController
self.__allavailprops = None
def __str__(self):
return "{} at {}".format(self.__class__.__name__,self.uri())
def uri(self):
return self._uri
def getBoringDict(self):
""" Return the dictionary this class writes and read on """
return self.valuedict
def valueOf(self, relative_uri):
""" Return the value of a child element.
relative_uri is the uri (a dictionary with the path like
["prop1", "listprop", 1, "prop2"])
which is relative to this PropertyData.
E.g. if we are in PropertyData of prop1 the list would
look like ["listprop", 1, "prop2"].
If no such element found None will be returned.
"""
if relative_uri == []:
return None
for el in self.givenProperties(): #type: PropertyDataObject
if el.info().name() == relative_uri[0]:
return el.valueOf(relative_uri[1:])
return None
def signalController(self):
""" Returns an object which can delay signals.
It will delay signals for all docks in this hierarchy.
That means that the child and parents DataObjects share this signalController
and locking it will cause every object here to delay its signals.
"""
return self._signalcontroller
def __buildProperty(self, name, meta):
meta = meta #type: info.Parameter
uri = self.uri()+[name]
info = self._infobuilder.buildInfo(name, meta, uri)
PropertyType = propertyObjectForInfo(info)
res = PropertyType(info, self.valuedict, name, self._infobuilder, uri=uri, signalcontroller = self._signalcontroller)
res.someChildPropertyChanged.connect(self.someChildPropertyChanged)
return res
def __buildProperties(self):
res = []
for el in self.valuedict:
res.append(self.__buildProperty(el, self.metadict[el]))
return res
def giveProperty(self, name, value):
""" Add the property "name" to the given one
and sets its value to "value".
value should be the raw value, not something like
PropertyDataObject.
If the property is already given, it gets overwritten
"""
item = self.getChildDataObject(name)
allprops = self.allAvailableProperties()
if not name in allprops:
raise ValueError("Unexpected key {} in {}".format(name,self._uri))
if not checkIsType(value,allprops[name].typeString()):
raise ValueError("Expected {} found type {}".format(allprops[name].typeString(), type(value)))
if item:
item.setValue(value)
return
self.valuedict[name] = value
if self._buildProps is None:
self.__buildProperties()
else:
newProperty = self.__buildProperty(name, self.metadict[name])
self._buildProps.append(newProperty)
self._emitGiven(newProperty, name)
def givePropertyDefault(self, name):
""" Like giveProperty but instead of give an explicit value the property will
be initialize with a default value (info.prototype() of this property).
If the value name is already given, nothing happen.
"""
if item in self:
return
props = self.allAvailableProperties() #type: Dict[str,PropertyInfo]
if not name in props.keys():
raise ValueError("Property {} is not a valid one".format(name))
return self.giveProperty(name, props[name].prototype())
def _emitGiven(self, newProp, name):
""" Emit someChildPropertyChanged-signal for giving a property with support for SignalController """
if self._signalcontroller:
self._signalcontroller.emitLater(self.propertyGotGiven, newProp)
self._signalcontroller.emitLater(self.someChildPropertyChanged, self.uri()+[name], ReasonGiven(name))
else:
self.propertyGotGiven.emit(newProp)
self.someChildPropertyChanged.emit(self.uri()+[name], ReasonGiven(name))
def ungiveProperty(self, name):
""" Unset the property with the given name """
del self.valuedict[name]
for idx, el in enumerate(self._buildProps):
if el.info().name() == name:
del self._buildProps[idx]
break
self._emitUngiven(name)
def _emitUngiven(self, name):
""" Emit someChildPropertyChanged-signal for ungiving a property with support for SignalController """
if self._signalcontroller:
self._signalcontroller.emitLater(self.propertyGotUnGiven, name)
self._signalcontroller.emitLater(self.someChildPropertyChanged, self.uri()+[name], ReasonUngiven(name))
else:
self.propertyGotUnGiven.emit(name)
self.someChildPropertyChanged.emit(self.uri()+[name], ReasonUngiven(name))
def allAvailableProperties(self):
""" Returns a dictionary, which looks like this:
{ "name_of_property1": PropertyInfo1,...}
"""
if self.__allavailprops: # Lazy initialization
return self.__allavailprops
res = {}
for key in self.metadict: # type: str
res[key] = self._infobuilder.buildInfo(key, self.metadict[key], self.uri()+[key])
self.__allavailprops = res
return res
def givenProperties(self):
""" Returns a list with infos about all properties:
Structure looks like this:
[
PropertyDataObject(), ...
]
"""
if self._buildProps is None:
self._buildProps = self.__buildProperties()
return self._buildProps
def getChildDataObject(self, key):
""" Return the child PropertyDataObject of the given key """
props = self.givenProperties()
for element in props: # type: PropertyDataObject
if element.info().name() == key:
return element
return None
def __getitem__(self, key):
""" Returns the value of the child object with the given key, if it's given """
el = self.getChildDataObject(key)
if el is None:
raise ValueError("PropertyData does not have the value {} defined at {}".format(key, self._uri))
return el.listGetValue()
def __setitem__(self, key, val):
""" Sets the property key to the value val, if it is a primitive (not a list or another group) """
allProps = self.allAvailableProperties()
if not key in allProps:
raise ValueError("Property {} cannot be set this property is not valid".format(key))
info = self.allAvailableProperties()[key]
if info.typeString() in [ListType, GroupType]:
# We can not change PropertyData e.g. because
# they are builded by a fix procedure
raise ValueError("Can only change primitive types through this function, not list or groups")
self.giveProperty(key,val)
def __contains__(self,key):
""" Return true iff the value key is a given property """
props = self.givenProperties()
for element in props: #type: PropertyDataObject
if element.info().name() == key:
return True
return False
def __delitem__(self, key):
self.ungiveProperty(key)
def __iter__(self):
props = self.givenProperties()
for element in props: #type: PropertyDataObject
yield element.info().name() Barista/gui/main_window/docks/properties/property_widgets/generic_info.py 0000664 ڊ0011117 00000016545 13265130053 027410 0 ustar klemms o0cvpr import copy
from gui.main_window.docks.properties.property_widgets import data
from gui.main_window.docks.properties.property_widgets.types import *
class GenericBuilderException(Exception):
pass
class GenericInfoBuilder(data.PropertyInfoBuilder):
""" This class can can build infos from a description which
is a dictionary which looks like this:
{
"type": GroupType, # type information, not optional
"name": "Bla", # name of this property, not optional
"prototype": {}, # a value of which a copy will be a default value, not optional. If it is a function, the function get called, but nothing will be copied then
"entries": [ # if Group, we have need the infos of the entries, not optional for this type
{
"name": "BlaSub",
"type": IntType, # type is int
"prototype": 0 # the default value
},
{
"name": "BlaSubList",
"type": ListType, # A list
"prototype": []
"entry": { # A list contains homogeneous entries
# Which should be described here
"name": "ListEntry",
"type": IntType,
"prototype": 0
}
# alternatively to entry we can set
# "typeEntry": IntType
},
{
"name": "BlaSubDict",
"type": DictType, # A Dictionary
"prototype": {}
"entry": { # A dictionary contains homogeneous values
# (and String-Keys)
# Which should be described here
"name": "DictEntry",
"type": FloatType,
"prototype": 0.2
}
# alternatively to entry we can set
# "typeEntry": FloatType
},
{
"name": "SomeEnum",
"type": EnumType,
"prototype": "A",
"enumOptions": ["A","B"]
},
{ # If we want to redirect infos to another
# builder, we can use the "builder"-attribute.
# With the "info"-entry, this class will represents the info.
# If the argument is a function, the function get called
# If info and builder are set, there is no need for other keys
"info": SomeOtherInfoClass
# Builder should always been set if info is set and the type in info is GroupType
"builder": SomeOtherInfoBuilder,
# A function changes the uri for
# the builder.
# If not set the builder gets the whole uri
"urifunc": lambda uri: uri[:3]
}
]
# Other available entries have the same name
# as the function in PropertyInfo class
}
"""
def __init__(self, infodescritpion):
self._infodescritpion = infodescritpion
def buildInfo(self, name, protoparameter, uri=None):
descr = self._infodescritpion
def req(name):
try:
return descr[name]
except KeyError:
raise GenericBuilderException("Key '{}' needed for {}".format(name, descr))
for i, uripart in enumerate(uri):
if "info" in descr:
descrtype = descr["info"].typeString()
else:
descrtype = req( "type" )
if not descrtype in [ListType, GroupType, DictType]:
raise GenericBuilderException("Cannot build info for {} because {} is a primitive".format(uri, uri[:i+1]))
if "builder" in descr:
urifunc = lambda x: x
if "urifunc" in descr:
urifunc = descr["urifunc"]
return descr["builder"].buildInfo(name, protoparameter, urifunc(uri))
elif descrtype == GroupType:
entries = descr["entries"]
for entry in entries:
try:
if "info" in entry:
if entry["info"].name() == uri[i]:
descr = entry
break
elif entry["name"] == uri[i]:
descr = entry
break
except KeyError:
raise GenericBuilderException("Key 'name' needed for {}".format(entry))
elif descrtype in [ListType, DictType]:
descr = req( "entry" )
if "info" in descr:
return descr["info"]
else:
return GenericInfoWrapper(descr)
def buildRoot(self):
if "info" in self._infodescritpion:
return self._infodescritpion["info"]
return GenericInfoWrapper(self._infodescritpion)
class GenericInfoWrapper(data.PropertyInfo):
""" Builds PropertyInfo on a info description (see the dictionaries GenericInfoBuilder) """
def __init__(self, description):
self._description = description
def name(self):
return self.__requiredType("name")
def typeEntryString(self):
if "typeEntry" in self._description:
return self._description["typeEntry"]
return self.typeString()
def typeString(self):
return self.__requiredType("type")
def __requiredType(self, name):
# Make own exception if key does not exist
try:
return self._description[name]
except KeyError:
raise GenericBuilderException("Entry {} needed in info-description".format(name))
def __entryOr(self, name, default):
# Returns default if name is not in self._description
if name in self._description:
return self._description[name]
return default
def entryInfo(self):
if "entry" in self._description:
entry = self._description["entry"]
if "info" in entry:
return entry["info"]
return GenericInfoWrapper(entry)
else:
return super(GenericInfoWrapper, self).entryInfo()
def enumOptions(self):
return self.__entryOr("enumOptions", [])
def deprecated(self):
return self.__entryOr("deprecated", False)
def description(self):
return self.__entryOr("description", "")
def prototype(self):
prototype = self.__requiredType("prototype")
if type(prototype) == function:
return prototype()
return copy.deepcopy(prototype)
def subparameter(self):
if self.typeString() == GroupType:
subdescr = self.__requiredType("entries")
res = {}
for descr in subdescr:
# We set them to None, because GenericInfoBuilder
# does not handle protoparameter directly.
# So every value should be fine
if "info" in descr:
res[descr["info"].name()] = None
elif "name" in descr:
res[descr["name"]] = None
else:
raise GenericBuilderException("{} needs a name".format(descr))
return res
return []
def isEditable(self):
return self.__entryOr("isEditable", True)
def isRequired(self):
return self.__entryOr("isRequired", True)
Barista/gui/main_window/docks/properties/property_widgets/layer_info.py 0000664 ڊ0011117 00000006772 13265130053 027111 0 ustar klemms o0cvpr import backend.caffe.proto_info as info
from gui.main_window.docks.properties.property_widgets import data
from gui.main_window.docks.properties.property_widgets.types import *
class LayerPropertyInfoBuilder(data.PropertyInfoBuilder):
def buildInfo(self, name, protoparameter, uri):
if uri == []:
return LayerParamInfo(name, protoparameter)
return PropertyInfo(name, protoparameter, uri)
def buildRootInfo(self, name, type):
""" Builds a info object for the whole layer
type should be an instance of ProtoInfo.LayerType
name is the name which the info type should have
"""
params = type.parameters()
return self.buildInfo(name, params, [])
class PropertyInfo(data.PropertyInfo):
""" This Class gives infos about a type in properties """
def __init__(self, name, meta,uri):
self._name = name #type: str
self.meta = meta #type: info.Parameter
self.uri = uri #type: list
def typeEntryString(self):
""" Give the type as String.
If the type is a list, returns the type of the entries.
"""
if self.meta.isInt():
return IntType
if self.meta.isDouble() or self.meta.isFloat():
return FloatType
if self.meta.isString():
return StringType
if self.meta.isEnum():
return EnumType
if self.meta.isParameterGroup():
return GroupType
if self.meta.isBool():
return BoolType
def typeString(self):
""" Give the type as String (see StringType, IntType,...)"""
if self.meta.isRepeated():
return ListType
return self.typeEntryString()
def enumOptions(self):
return self.meta.availableValues()
def name(self):
""" Return the name of this property """
return self._name
def deprecated(self):
""" Return true iff this property is deprecated """
return self.meta.isDeprecated()
def description(self):
""" Return a decription of this property """
return self.meta.description()
def prototype(self):
""" Returns a raw value of a default type """
default = self.meta.defaultValue()
if default is None:
return data.prototypeOfType(self)
return default
def subparameter(self):
return self.meta.parameter()
def isEditable(self):
uri = self.uri
if uri and len(uri) > 0:
if uri in [["type"]]: return False
if uri in [["bottom"]]: return False
if uri in [["top"]]: return False
# if uri[0] in ["bottom"]: return False
return True
def isRequired(self):
uri = self.uri
if uri and len(uri) > 0:
return uri in [["name"], ["type"]]
return False
class LayerParamInfo(data.PropertyInfo):
""" Give infos about the root parameters of a layer"""
def __init__(self, name, parameters):
self._name = name
self.params = parameters
def typeEntryString(self):
return self.typeString()
def typeString(self):
return GroupType
def enumOptions(self):
return []
def name(self):
return self._name
def deprecated(self):
return False
def description(self):
return ""
def prototype(self):
return {}
def subparameter(self):
return self.params
def isEditable(self):
return False
def isRequired(self):
return False
Barista/gui/main_window/docks/properties/property_widgets/network_dict_info.py 0000664 ڊ0011117 00000013136 13265130053 030461 0 ustar klemms o0cvpr import backend.caffe.proto_info as info
from gui.main_window.docks.properties.property_widgets import data
from gui.main_window.docks.properties.property_widgets import layer_info
from gui.main_window.docks.properties.property_widgets.types import *
class NetworkDictInfoBuilder(data.PropertyInfoBuilder):
""" Builds info docks for the network-part of the state-dictionary """
def __init__(self, netdict):
if type(netdict) == dict:
self.netdictfunc = lambda: netdict
else:
self.netdictfunc = netdict
self.layerinfo = layer_info.LayerPropertyInfoBuilder()
def buildInfo(self, name, protoparameter, uri=None):
if uri == []:
return NetworkDictInfo(name, protoparameter)
elif uri == ["layers"]:
return LayersInfo("layers", LayerIdInfo(""))
elif uri[0] == "layers":
id = uri[1]
layer = self.netdictfunc()["layers"][id]
if len(uri) == 2:
return LayerIdInfo(id)
if uri[2] == "type":
return LayerTypeInfo()
if uri[2] == "parameters":
if uri == ["layers",id,"parameters"]:
return self.layerinfo.buildRootInfo(name, layer["type"])
else:
return self.layerinfo.buildInfo(name, protoparameter, uri[3:])
elif uri == ["layerOrder"]:
return LayerOrderInfo("layerOrder")
else:
return NetworkInfo(name, protoparameter, uri)
def buildRootInfo(self, name=""):
""" Builds the information about "network" in the state dictionary """
prot = info.CaffeMetaInformation().availableNetParameters().copy()
del prot["layers"]
del prot["layer"]
prot["layers"] = "LayerDict"
prot["layerOrder"] = "LayerOrder"
return self.buildInfo(name,prot,[])
class LayersInfo(data.PropertyInfo):
""" Give infos about the entry 'layers' in the network dictionary """
def __init__(self, name, layerIdInfo):
self._name = name
self.paramInfo = layerIdInfo
def entryInfo(self):
return self.paramInfo
def typeString(self):
return DictType
def enumOptions(self):
return []
def name(self):
return self._name
def deprecated(self):
return False
def description(self):
return ""
def prototype(self):
return {}
def subparameter(self):
""" Returns the subparameter of an entry in this dictionary (the layer id subparemter = "parameters","type")"""
return self.paramInfo.subparameter()
def isEditable(self):
return True
def isRequired(self):
return True
class LayerOrderInfo(data.PropertyInfo):
def __init__(self, name):
self._name = name
def typeEntryString(self):
return StringType
def typeString(self):
return ListType
def enumOptions(self):
return []
def name(self):
return self._name
def deprecated(self):
return False
def description(self):
return ""
def prototype(self):
return []
def subparameter(self):
return {}
def isEditable(self):
return True
def isRequired(self):
return True
class LayerIdInfo(data.PropertyInfo):
""" Give infos about a layer (with type and parameters) in network dictionary """
def __init__(self, id):
self.id = id
def typeEntryString(self):
return self.typeString()
def typeString(self):
return GroupType
def enumOptions(self):
return []
def name(self):
return self.id
def deprecated(self):
return False
def description(self):
return ""
def prototype(self):
return {"type": None, "parameters": {}}
def subparameter(self):
return {"type": "type", "parameters": "parameters"}
def isEditable(self):
return False
def isRequired(self):
return False
class LayerTypeInfo(data.PropertyInfo):
""" Give infos about the type-attribute of a layer network dictionary"""
def __init__(self):
pass
def typeEntryString(self):
return self.typeString()
def typeString(self):
return UnknownType
def enumOptions(self):
return []
def name(self):
return "type"
def deprecated(self):
return False
def description(self):
return ""
def prototype(self):
return None
def subparameter(self):
return {}
def isEditable(self):
return False
def isRequired(self):
return True
class NetworkInfo(layer_info.PropertyInfo):
""" Give infos about a non-layer parameter in the network-dictionary"""
def __init__(self, name, meta, uri):
self._name = name
self.meta = meta
self.uri = uri
def isEditable(self):
return True
def isRequired(self):
return self.uri in [[]]
class NetworkDictInfo(data.PropertyInfo):
""" Give infos about the root of the network dictionary"""
def __init__(self, name, prot):
self.prot = prot
self._name = name
def typeEntryString(self):
return self.typeString()
def typeString(self):
return GroupType
def enumOptions(self):
return []
def name(self):
return self._name
def deprecated(self):
return False
def description(self):
return ""
def prototype(self):
return {}
def subparameter(self):
return self.prot
def isEditable(self):
return True
def isRequired(self):
return True
Barista/gui/main_window/docks/properties/property_widgets/program_state_info.py 0000664 ڊ0011117 00000006467 13265130053 030645 0 ustar klemms o0cvpr import backend.caffe.dict_helper as helper
from gui.main_window.docks.properties.property_widgets import network_dict_info
from gui.main_window.docks.properties.property_widgets import solver_info
from gui.main_window.docks.properties.property_widgets.generic_info import GenericInfoBuilder
from gui.main_window.docks.properties.property_widgets.types import *
def defaultState():
net = helper.bareNet("Unnamed")
return {
"network": net,
"position": {},
"selection": [],
"solver": {},
"hidden_connections": {}
}
class ProgramStateInfoBuilder(GenericInfoBuilder):
def __init__(self, statedict):
self.statedict = statedict
self.netinfo = network_dict_info.NetworkDictInfoBuilder(lambda: self.statedict["network"])
self.solverinfo = solver_info.SolverPropertyInfoBuilder()
infodescription = {
"type": GroupType,
"name": "",
"prototype": defaultState,
"entries": [
{
"info": self.netinfo.buildRootInfo("network"),
"builder": self.netinfo,
"urifunc": lambda uri: uri[1:]
},
{
"name": "position",
"type": DictType,
"prototype": [],
"entry": {
"name": "coord",
"type": ListType,
"prototype": [0,0],
"typeEntry": IntType
}
},
{
"name": "selection",
"type": ListType,
"prototype": [],
"typeEntry": StringType
},
{
"info": self.solverinfo.buildRootInfo("solver"),
"builder": self.solverinfo,
"urifunc": lambda uri: uri[1:]
},
{
"name": "hidden_connections",
"type": ListType,
"prototype": [],
"isRequired": False,
"entry": {
"name": "hidden_connection",
"type": GroupType,
"prototype": {},
"entries": [
{
"name": "topLayerId",
"type": StringType,
"prototype": ""
},
{
"name": "topBlobIdx",
"type": IntType,
"prototype": 0
},
{
"name": "bottomLayerId",
"type": StringType,
"prototype": ""
},
{
"name": "bottomBlobIdx",
"type": IntType,
"prototype": 0
}
]
}
}
]
}
GenericInfoBuilder.__init__(self,infodescription)
Barista/gui/main_window/docks/properties/property_widgets/solver_info.py 0000664 ڊ0011117 00000005212 13265130053 027273 0 ustar klemms o0cvpr import backend.caffe.proto_info as info
from gui.main_window.docks.properties.property_widgets import data
from gui.main_window.docks.properties.property_widgets import layer_info
from gui.main_window.docks.properties.property_widgets.types import *
class SolverPropertyInfoBuilder(data.PropertyInfoBuilder):
""" Builds info docks for the solver part of the state dictionary """
def buildInfo(self, name, protoparameter, uri=None):
if uri == []:
return SolverRootInfo(name, protoparameter)
return PropertyInfo(name, protoparameter, uri)
def buildRootInfo(self, name=""):
""" Builds "solver" info object of the state dictionary """
from backend.caffe.path_loader import PathLoader
proto = PathLoader().importProto()
protosolver = proto.SolverParameter()
descr = info.ParameterGroupDescriptor(protosolver)
params = descr.parameter().copy()
return self.buildInfo(name,params, [])
class SolverRootInfo(data.PropertyInfo):
""" Represent the info about the parameters of a solver"""
def __init__(self, name, params):
self._name = name
self.params = params
def typeEntryString(self):
return self.typeString()
def typeString(self):
return GroupType
def enumOptions(self):
return []
def name(self):
return self._name
def deprecated(self):
return False
def description(self):
return ""
def prototype(self):
return {}
def subparameter(self):
""" Returns the subparameter of an entry in this dictionary (the layer id subparemter = "parameters","type")"""
return self.params
def isEditable(self):
return True
def isRequired(self):
return True
class PropertyInfo(layer_info.PropertyInfo):
def __init__(self, name, meta, uri):
super(PropertyInfo,self).__init__(name,meta,uri)
def isEditable(self):
uri = self.uri
if uri and len(uri) > 0:
if uri in [["net"], ["net_param"]]: return False
return True
def isRequired(self):
uri = self.uri
if uri and len(uri) > 0:
if uri in [["net"]]: return True
return False
def typeString(self):
if self.uri == ["type"]:
return EnumType
return super(PropertyInfo, self).typeString()
def enumOptions(self):
if self.uri == ["type"]:
return info.CaffeMetaInformation().availableSolverTypes()
return super(PropertyInfo, self).enumOptions()
def subparameter(self):
params = super(PropertyInfo,self).subparameter() #type: dict
return params
Barista/gui/main_window/docks/properties/property_widgets/type_widgets.py 0000664 ڊ0011117 00000021733 13265130053 027463 0 ustar klemms o0cvpr from PyQt5 import QtCore, QtGui
from PyQt5 import QtWidgets
from PyQt5.QtCore import pyqtSignal
from gui.main_window.docks.properties.property_widgets.data import StringType, IntType, EnumType, FloatType, BoolType, PropertyDataObject
class TwoTypeWdg:
""" TwoTypeWdg is a class which defines setupInTableLayout
using exactly two widget:
* one label for the name
* one widget defines by its subclasses in _initValueWdg
Child classes can use self._parent to get the parent widget.
They can use self.data to get the PropertyDataObject
this Widget represents.
"""
def __init__(self, data, parent=None):
""" Build TwoTypeWdg with given parent widget
and the instance of PropertyDatObject named data.
The name of the name-label will be the value
of data.info().name().
"""
# super(TwoTypeWdg, self).__init__(parent)
self.data = data #type: PropertyDataObject
self._parent = parent
self.nameWdg = QtWidgets.QLabel(self.data.info().name(),parent)
self.valueWdg = self._initValueWdg()
self.valueWdg.setSizePolicy(QtWidgets.QSizePolicy(QtWidgets.QSizePolicy.Expanding, QtWidgets.QSizePolicy.Fixed))
self.nameWdg.setSizePolicy(QtWidgets.QSizePolicy(QtWidgets.QSizePolicy.Expanding, QtWidgets.QSizePolicy.Fixed))
font = self.nameWdg.font() #type: QtGui.QFont
# self.nameWdg.setAlignment(QtCore.Qt.AlignRight | QtCore.Qt.AlignVCenter)
if self.data.info().deprecated():
font.setStrikeOut(True)
self.nameWdg.setFont(font)
self.nameWdg.setToolTip(getTooltipString(self.data.info().description()))
self.valueWdg.setToolTip(getTooltipString(self.data.info().description()))
if not data.info().isEditable():
self.nameWdg.setEnabled(False)
self.valueWdg.setEnabled(False)
def setupInTableLayout(self, lay, row, remButton):
""" Setup this object into a QGridLayout
row is the specific row to insert the widgets
remButton is the Remove-Button which should
be inserted into the GridLayout
by this function, too
"""
lay = lay #type: QtWidgets.QGridLayout
lay.addWidget(self.nameWdg, row, 0,1,1)
lay.addWidget(self.valueWdg, row, 1,1,1)
lay.addWidget(remButton, row, 2,1,1)
def _initValueWdg(self):
""" This function should be overriden by every child
It should return the Widget which represents the value
of the PropertyDataObject.
"""
pass
class StringWdg(TwoTypeWdg):
""" A Line-Editor for edit simple text """
class Editor(QtWidgets.QLineEdit):
""" The LineEdit-Widget """
""" This signal is emitted every time the value of this prototxt_editor
should propagate changes.
For example not every text change emits this signals.
Only when the prototxt_editor gets deselected or enter is pressed
this signal is emitted.
"""
saveChange = pyqtSignal(str)
def __init__(self, text="", parent=None):
super(StringWdg.Editor, self).__init__(text,parent)
self.customContextMenuRequested.connect(self.openCtxMenu)
self.setContextMenuPolicy(QtCore.Qt.CustomContextMenu)
self.returnPressed.connect(self._checkForChangesAndEmitIf)
self.__textInData = text
def openCtxMenu(self,pos):
""" Add "From File" and "From Path" to the context menu """
menu = self.createStandardContextMenu() # type: QtWidgets.QMenu
menu.addSeparator()
menu.addAction(self.tr("From &File"), self.selectFile)
menu.addAction(self.tr("From &Path"), self.selectPath)
menu.exec_(self.mapToGlobal(pos))
def setTextWithChangeDetection(self, text):
""" This method should be use from other to set the text of
this widget. It updates the widget text but also updates
the variable represents the last text data.
So the function _checkForChangesAndEmitIf works.
"""
self.__textInData = text
self.setText(text)
def _checkForChangesAndEmitIf(self):
""" Check is the new text does not equal
the last one and emit signals if not equal
"""
if self.text() != self.__textInData:
self.saveChange.emit(self.text())
def focusOutEvent(self, event):
super(StringWdg.Editor, self).focusOutEvent(event)
self._checkForChangesAndEmitIf()
def selectFile(self):
""" When clicking 'From File' """
filename,_ = QtWidgets.QFileDialog.getOpenFileName(self)
if len(filename) == 0:
return
self.setText(filename)
def selectPath(self):
""" When clicking 'From Path' """
dirname = QtWidgets.QFileDialog.getExistingDirectory(self)
if len(dirname) == 0:
return
self.setText(dirname)
def _initValueWdg(self):
wdg = StringWdg.Editor(self.data.value(),self._parent)
wdg.saveChange.connect(lambda: self.data.setValue(wdg.text()))
wdg.childEvent
self.data.propertyChanged.connect(wdg.setTextWithChangeDetection)
return wdg
class IntWdg(TwoTypeWdg):
""" A Spinbox which represent integer values """
# The maximal and minimal values of an Integer
MAX_RANGE = -pow(2, 31), pow(2, 31-1)
class SpinBox(QtWidgets.QSpinBox):
""" The SpinBox widget """
def __init__(self, parent=0):
super(IntWdg.SpinBox, self).__init__(parent)
self.setFocusPolicy(QtCore.Qt.StrongFocus )
# idea from http://stackoverflow.com/questions/5821802/qspinbox-inside-a-qscrollarea-how-to-prevent-spin-box-from-stealing-focus-when
def wheelEvent(self, event):
if not self.hasFocus():
event.ignore()
else:
QtWidgets.QSpinBox.wheelEvent(self, event)
def _initValueWdg(self):
wdg = IntWdg.SpinBox(self._parent)
wdg.setRange(*self.MAX_RANGE)
wdg.setValue(self.data.value())
wdg.valueChanged.connect(self.data.setValue)
self.data.propertyChanged.connect(wdg.setValue)
return wdg
class FloatWdg(TwoTypeWdg):
""" A Spinbox which represent float values """
MAX_RANGE = -pow(2, 31), pow(2, 31-1)
class SpinBox(QtWidgets.QDoubleSpinBox):
""" The Spinbox Widget """
def __init__(self, parent=0):
super(FloatWdg.SpinBox, self).__init__(parent)
self.setFocusPolicy(QtCore.Qt.StrongFocus )
# self.setDecimals(323)
self.setDecimals(15)
self.setSingleStep(0.05)
# idea from http://stackoverflow.com/questions/5821802/qspinbox-inside-a-qscrollarea-how-to-prevent-spin-box-from-stealing-focus-when
def wheelEvent(self, event):
if not self.hasFocus():
event.ignore()
else:
QtWidgets.QDoubleSpinBox.wheelEvent(self, event)
def textFromValue(self, value):
""" Change the text of this SpinBox such that the widget
does not show unneeded zeros after the decimal point
"""
stringValue = super(FloatWdg.SpinBox, self).textFromValue(value)
while len(stringValue) > 2 and stringValue[-1] == "0" and stringValue[-2] != ",":
stringValue = stringValue[0:-1]
return stringValue
def _initValueWdg(self):
wdg = FloatWdg.SpinBox(self._parent)
wdg.setRange(*self.MAX_RANGE)
wdg.setValue(self.data.value())
wdg.valueChanged.connect(self.data.setValue)
self.data.propertyChanged.connect(wdg.setValue)
return wdg
class EnumWdg(TwoTypeWdg):
""" Combobox for Enums """
def _initValueWdg(self):
wdg = QtWidgets.QComboBox(self._parent)
wdg.addItems(self.data.info().enumOptions())
wdg.currentTextChanged.connect(self.data.setValue)
self.data.propertyChanged.connect(wdg.setCurrentText)
wdg.setCurrentText(self.data.value())
return wdg
class BoolWdg(TwoTypeWdg):
""" Checkbox for Bools"""
def _initValueWdg(self):
wdg = QtWidgets.QCheckBox(self._parent)
wdg.stateChanged.connect(lambda s: self.data.setValue(wdg.isChecked()))
self.data.propertyChanged.connect(wdg.setChecked)
wdg.setChecked(self.data.value())
return wdg
# Export Widget
WdgCatalog={
StringType: StringWdg,
IntType: IntWdg,
FloatType: FloatWdg,
EnumType: EnumWdg,
BoolType: BoolWdg,
}
def getTooltipString(description):
"""Ensures that tooltips are shown with newlines. """
newDescription = ""
if not description == "":
newDescription = ""
newDescription += description
newDescription += ""
return newDescription
Barista/gui/main_window/docks/properties/property_widgets/types.py 0000664 ڊ0011117 00000001737 13265130053 026122 0 ustar klemms o0cvpr
StringType = "string"
IntType = "int"
FloatType = "float"
EnumType = "enum"
GroupType = "group"
ListType = "list"
BoolType = "bool"
DictType = "dict"
UnknownType = "unknown"
def checkIsType(instance, typestring):
""" Checks if instance match the type expected from typestring.
E.g. checkIsType(22, IntType) == True
checkIsType(true, StringType) == False
"""
if typestring in [StringType, EnumType]:
return type(instance) in [str, unicode]
if typestring == IntType:
return type(instance) in [int, long]
if typestring == FloatType:
return type(instance) in [int, long, float]
if typestring in [GroupType, DictType]:
return type(instance) == dict
if typestring == BoolType:
return type(instance) == bool
if typestring in [ListType]:
return type(instance) == list
if typestring == UnknownType:
return True
raise ValueError("Unhandable typestring {}".format(typestring))
Barista/gui/main_window/docks/properties/property_widgets/widgets.py 0000664 ڊ0011117 00000035616 13265130053 026427 0 ustar klemms o0cvpr import platform
from PyQt5 import QtCore, QtGui
from PyQt5 import QtWidgets
from PyQt5.QtCore import pyqtSignal
from gui.main_window.docks.properties.property_widgets.data import PropertyDataObject, PropertyDataListObject
from gui.main_window.docks.properties.property_widgets.type_widgets import WdgCatalog
from gui.main_window.docks.properties.property_widgets.types import *
def clearLayout(lay):
""" Removes every widget in the given layout """
for i in reversed(range(lay.count())):
item = lay.itemAt(i) # type: QtWidgets.QLayoutItem
lay.removeItem(item)
wdg = item.widget()
if wdg is None:
childLay = item.layout()
clearLayout(childLay)
else:
wdg.setParent(None)
wdg.deleteLater()
class PropertyEditWdg(QtWidgets.QScrollArea):
""" This is the main widget for showing the properties.
It should be used in the dock element for example.
"""
def __init__(self, parent=None, data = None):
""" Construct this object with the parent (a widget)
and the data (PropertyData/-Object).
Through this widget the user can manipulate
the data. data can be None, so there is no
data set.
"""
super(PropertyEditWdg, self).__init__(parent)
self.setWidgetResizable(True)
self.content = None
if data:
self.setData(data)
def setData(self, data):
""" Set the data (may replace old one) with the given one """
self.content = GroupWdg(data)
# lay = QtWidgets.QVBoxLayout()
# lay.addWidget(val)
# self.setLayout(lay)
self.setWidget(self.content)
def disableEditing(self, disable):
""" Disable Editing of the data """
if self.content:
self.content.setDisabled(disable)
class RepeatedWrapper(QtWidgets.QGroupBox):
""" A Widget for showing a list of values """
def __init__(self, data, parent=None):
super(RepeatedWrapper, self).__init__(parent)
# Layout to add the child widgets
# Have to be a GridLayout.
self._childlay = QtWidgets.QGridLayout()
self._childlay.setHorizontalSpacing(2)
# Layout for the top-remove button
self._topLay = QtWidgets.QHBoxLayout()
# Layout of the whole object
self._lay = QtWidgets.QVBoxLayout()
self._lay.addLayout(self._topLay)
self._lay.addLayout(self._childlay)
self._lay.addStretch()
# self._lay.setContentsMargins(1,1,1,1)
self.setLayout(self._lay)
self.data = data #type: PropertyDataListObject
self.nameLabel = QtWidgets.QLabel(data.info().name())
self.setToolTip(getTooltipString(self.data.info().description()))
# \/ exclusive value
self.lastChildRow = self._buildChilds(self.data.value())
self.data.propertyChanged.connect(self._buildChilds)
# Layout for the add-Button
bottomLay = QtWidgets.QHBoxLayout()
self._lay.addLayout(bottomLay)
bottomLay.addWidget(self.nameLabel,self.lastChildRow)
bottomLay.addStretch()
addButton = QtWidgets.QToolButton(self)
addButton.clicked.connect(self.addOne)
addButton.setText("+")
bottomLay.addWidget(addButton,self.lastChildRow)
def _clear(self):
""" Clear the Layout so there is no widget anymore """
clearLayout(self._childlay)
self.lastChildRow = 0
def _buildChilds(self, childs):
""" Build the widget for every entry in the list """
self._clear()
# if remove button is clicked
def removeCallback(idx):
return lambda: self.data.removeObject(idx)
i = 1 # Current entry-idx in the GridLayout
for (idx,element) in enumerate(childs): #type: (int,PropertyDataObject)
# Build the widget
wdg = WdgCatalog[element.info().typeString()](element,self)
# The remove button
remButton = QtWidgets.QToolButton(self)
trashIcon = QtGui.QPixmap('resources/trash.png')
remButton.setIcon(QtGui.QIcon(trashIcon))
remButton.clicked.connect(removeCallback(idx))
wdg.setupInTableLayout(self._childlay,idx,remButton)
i += 1
self.nameLabel.setVisible(len(childs) == 0)
return i
def setupInTableLayout(self,lay, row, remButton):
""" Install this widget in the QGridLayout lay """
def hline():
line = QtWidgets.QFrame(self)
line.setFrameShape(QtWidgets.QFrame.HLine)
line.setFrameShadow(QtWidgets.QFrame.Sunken)
return line
# Add the remove button to the top
# The remove button is given as an argument
# to this function.
self._topLay.addWidget(hline())
self._topLay.addWidget(remButton)
self._topLay.addWidget(hline())
# And this widget to the GridLayout
lay.addWidget(self, row,0,1,3)
def addOne(self):
""" Add a new entry to the list using
the default value (prototypeValue)
of the PropertyInfo of the data
"""
self.data.pushBack(self.data.prototypeValue())
class GroupWdg(QtWidgets.QGroupBox):
""" Widget for a group of values """
class Model(QtCore.QAbstractListModel):
""" List Model for the autocompletion in 'New Property' Text-Field """
def __init__(self, data, parent=None):
super(GroupWdg.Model, self).__init__(parent)
self.data = data #type: PropertyData
def newProperties(self):
""" Return a list of properties which are not
set in the GroupData
"""
all = self.data.allAvailableProperties()
given = [prop.info().name() for prop in self.data.givenProperties()]
res = []
for name in all:
if not name in given:
res.append(all[name])
# sort properties alphabetically
res.sort(key=lambda x: x.name())
# move deprecated properties to the end of the list
resFinal = []
resDeprecated = []
for property in res:
if property.deprecated():
resDeprecated.append(property)
else:
resFinal.append(property)
resFinal.extend(resDeprecated)
return resFinal
def rowCount(self, idx):
return len(self.newProperties())
def data(self, idx, role):
row = idx.row()
prop = self.newProperties()[row]
if role == QtCore.Qt.DisplayRole:
return prop
if role ==QtCore.Qt.EditRole:
return prop.name()
return None
class Delegate(QtWidgets.QStyledItemDelegate):
""" The visual Item of a entry in the autocompletion """
def __init__(self, parent=None):
super(GroupWdg.Delegate,self).__init__(parent)
def paint(self, painter, option, idx):
# super(Delegate, self).paint(painter, option, idx)
data = idx.data()
painter = painter #type: QtGui.QPainter
painter.save()
# If the mouse is over this item
if option.state & QtWidgets.QStyle.State_Selected:
painter.fillRect(option.rect,option.palette.highlight())
painter.setBrush(option.palette.highlightedText())
else:
painter.fillRect(option.rect,option.palette.base())
font = painter.font() #type: QtGui.QFont
# Strike out if the data is deprecated
font.setStrikeOut(data.deprecated())
painter.setFont(font)
painter.drawText(option.rect, QtCore.Qt.AlignLeft | QtCore.Qt.AlignVCenter, data.name())
painter.restore()
def __init__(self, data, parent=None):
super(GroupWdg, self).__init__(parent)
self.name = ""
self._topLay = QtWidgets.QHBoxLayout()
self._editable = True
# If data is a ProeprtyDataObject we get more
# infos which we can use.
if isinstance(data, PropertyDataObject):
self.setToolTip(getTooltipString(data.info().description()))
self.name = data.info().name()
self._editable = data.info().isEditable()
data = data.value()
# Titel der Group-Box
self.setTitle(self.name)
self.data = data #type: PropertyData
self.data.propertyGotGiven.connect(self._addProperty)
self.data.propertyGotUnGiven.connect(self._buildChilds)
lay = QtWidgets.QVBoxLayout()
self.childLay = QtWidgets.QGridLayout()
# self.childLay.setContentsMargins(2,2,2,2)
lay.setContentsMargins(3,3,3,3)
self.childLay.setHorizontalSpacing(2)
lay.addLayout(self._topLay)
lay.addLayout(self.childLay)
lay.addStretch()
# \/ exclusive
self.lastChildRow = 0
self._buildChilds()
lay.addLayout(self._buildAddLayout())
self.setLayout(lay)
self.wdgDict = {}
def _addProperty(self, property):
""" Add the property to this group when it become given.
property is an instance of PropertyDataObject or PropertyData
"""
def removeCallback(name):
return lambda: self._removeElement(name)
# We create a widget depending on the type of the property.
wdg = WdgCatalog[property.info().typeString()](property,self)
# Remove Button
button = QtWidgets.QToolButton(self)
trashIcon = QtGui.QPixmap('resources/trash.png')
button.setIcon(QtGui.QIcon(trashIcon))
wdg.setupInTableLayout(self.childLay, self.lastChildRow, button)
button.setEnabled(not property.info().isRequired())
name = property.info().name()
button.clicked.connect(removeCallback(name))
self.lastChildRow+=1
self.wdgDict[property.info().name()] = wdg
class CompleteLineEdit(QtWidgets.QLineEdit):
""" The LineEdit with autocompletion """
""" Signal is emitted when a new text has
been chosen by the user """
gotChoice = pyqtSignal(str)
def __init__(self, parent=None):
super(GroupWdg.CompleteLineEdit, self).__init__(parent)
self.returnPressed.connect(lambda: self.gotChoice.emit(self.text()))
self.gotChoice.connect(self._clear)
def _clear(self):
""" Clear completer and text """
self.completer().setCompletionPrefix("")
self.clear()
def focusInEvent(self,ev):
""" Show completion if the prototxt_editor gets in focus """
super(GroupWdg.CompleteLineEdit, self).focusInEvent(ev)
self.showComplete()
def setCompleter(self, completer):
super(GroupWdg.CompleteLineEdit,self).setCompleter(completer)
# completer.activated.connect(self.gotChoice)
def showComplete(self):
""" Shows the completion if an completer is set.
It give the right prefix to the completion.
"""
if self.completer() is None:
return
self.completer().setCompletionPrefix(self.text())
self.completer().complete()
def keyPressEvent(self, event):
"""
Ctrl+Space should show the completer
"""
super(GroupWdg.CompleteLineEdit,self).keyPressEvent(event)
ctrl = QtCore.Qt.ControlModifier
# Qt replace Ctrl with Cmd on macOS but we do not want that
if platform.system() == "Darwin":
ctrl = QtCore.Qt.MetaModifier
if event.modifiers() == ctrl and event.key() == QtCore.Qt.Key_Space:
self.showComplete()
def _addElement(self, name):
""" Add a new property initialized with the default (protoype) value.
If the name is invalid (e.g. is not a property for this group ),
the user gets a Popup which informs him
"""
newOne = self._model.newProperties()
for item in newOne:
if item.name() == name:
self.data.giveProperty(name,item.prototype())
return
label = QtWidgets.QLabel(self.tr("This property does not exists"))
label.setWindowFlags(QtCore.Qt.Popup)
label.move(self.mapToGlobal(self.editor.geometry().topLeft()+QtCore.QPoint(0,-10)))
label.show()
QtCore.QTimer.singleShot(2000, lambda: label.close())
def _buildAddLayout(self):
""" Build layout for the autocompletion-LineEdit"""
lay = QtWidgets.QHBoxLayout()
# Editor
self.editor = GroupWdg.CompleteLineEdit()
self.editor.setPlaceholderText(self.tr("New Property"))
# Model
self._model = GroupWdg.Model(self.data)
# Completer
completer = QtWidgets.QCompleter()
completer.setModel(self._model)
completer.setModelSorting(QtWidgets.QCompleter.CaseInsensitivelySortedModel)
completer.popup().setItemDelegate(GroupWdg.Delegate(self))
completer.setCompletionMode(QtWidgets.QCompleter.PopupCompletion)
self.editor.setCompleter(completer)
lay.addStretch()
lay.addWidget(self.editor)
# Signals
self.editor.gotChoice.connect(self._addElement)
self.editor.setEnabled(self._editable)
return lay
def setupInTableLayout(self,lay, row, remButton):
""" Setup this widget in parent QGridLayout """
def hline():
line = QtWidgets.QFrame(self)
line.setFrameShape(QtWidgets.QFrame.HLine)
line.setFrameShadow(QtWidgets.QFrame.Sunken)
return line
# Remove button at the top
self._topLay.addWidget(hline())
self._topLay.addWidget(remButton)
self._topLay.addWidget(hline())
lay.addWidget(self, row,0,1,3)
def _removeElement(self, name):
""" Remove the property with the given name from the data
and this Widget
"""
self.data.ungiveProperty(name)
def _clear(self):
""" Clears all children """
clearLayout(self.childLay)
self.lastChildRow = 0
self.wdgDict = {}
def _buildChilds(self):
""" Build the children widget which represent the
given properties of the PropertyDataGroupObject
"""
self._clear()
d = dict([(prop.info().name(), prop) for prop in self.data.givenProperties()])
keys = d.keys()
keys.sort()
for key in keys:
property = d[key]#type: PropertyDataObject
self._addProperty(property)
def getTooltipString(description):
"""Ensures that tooltips are shown with newlines. """
newDescription = ""
if not description == "":
newDescription = ""
newDescription += description
newDescription += ""
return newDescription
WdgCatalog[GroupType] = GroupWdg
WdgCatalog[ListType] = RepeatedWrapper
Barista/gui/main_window/node_editor/items/renderer/__init__.py 0000664 ڊ0011117 00000000000 13265130053 025004 0 ustar klemms o0cvpr Barista/gui/main_window/node_editor/items/renderer/connection_item_renderer.py 0000664 ڊ0011117 00000005667 13265130053 030340 0 ustar klemms o0cvpr from PyQt5.QtGui import QPainter, QPen
from gui.main_window.node_editor.node_editor_constants import Constants
class ConnectionItemRenderer:
""" Class is used to separate the drawing from the logic.
This class renders a given connection to a graphics scene """
def __init__(self, connection):
self.__connection = connection
def paint(self, painter):
# use antialiasing
painter.setRenderHint(QPainter.Antialiasing)
# connections, connected to layers with the include.phase parameter,
# have a different opacity than normal connections
opacity = Constants.itemOpacityInPhase
if not self.__connection.getNodeEditor().isCurrentPhase(self.__connection.getPhase()):
opacity = Constants.itemOpacityNotInPhase
# connections between a layer and a in-place working layer have a different color than normal connections
connectionColor = Constants.connectionItemColor
if self.__connection.getIsInPlace():
connectionColor = Constants.itemInPlaceColor
# set local color object alpha
connectionColor.setAlpha(opacity)
# draw highlight if the connection is selected
if self.__connection.isSelected():
selectedColor = Constants.selectedColor
selectedColor.setAlpha(opacity)
painter.setPen(QPen(selectedColor, Constants.connectionItemSelectionSize))
painter.drawPath(self.__connection.path())
pen = QPen(connectionColor, Constants.connectionItemSize)
# if the connection is hidden, draw a dashed line at the start and end of the connection
if self.__connection.getHidden():
dashes = []
# calculate the dashes at the start of the connection
for i in range(0, Constants.connectionItemHiddenDashCount - 1):
dashes.append(Constants.connectionItemHiddenDashSize / pen.width())
dashes.append(Constants.connectionItemHiddenDashSpace / pen.width())
dashes.append(Constants.connectionItemHiddenDashSize / pen.width())
# calculate the size of the space between the dashed start and dashed end
middleSpace = self.__connection.path().length()
middleSpace -= 2 * Constants.connectionItemHiddenDashCount * Constants.connectionItemHiddenDashSize
middleSpace -= 2 * (Constants.connectionItemHiddenDashCount - 1) * Constants.connectionItemHiddenDashSpace
dashes.append(middleSpace / pen.width())
# calculate the dashes at the end of the connection
for i in range(0, Constants.connectionItemHiddenDashCount):
dashes.append(Constants.connectionItemHiddenDashSize / pen.width())
dashes.append(Constants.connectionItemHiddenDashSpace / pen.width())
pen.setDashPattern(dashes)
# draw the connection
painter.setPen(pen)
painter.drawPath(self.__connection.path())
Barista/gui/main_window/node_editor/items/renderer/connector_item_renderer.py 0000664 ڊ0011117 00000004354 13265130053 030163 0 ustar klemms o0cvpr from PyQt5.QtCore import QRectF
from PyQt5.QtGui import QPen, QBrush
from gui.main_window.node_editor.node_editor_constants import Constants
class ConnectorItemRenderer:
""" Class is used to separate the drawing from the logic.
This class renders a given connector to a graphics scene """
def __init__(self, connector):
self.__connector = connector
# get the outer and inner size of the enclosing rect (inner size is the size of the hole)
outerSize = Constants.connectorItemOuterSize
innerSize = Constants.connectorItemInnerSize
# calculate the Qt rects centered at the center of the rect
self.__outerRect = QRectF(-(outerSize / 2), -(outerSize / 2),outerSize, outerSize)
self.__innerRect = QRectF(-(innerSize / 2), -(innerSize / 2), innerSize, innerSize)
def boundingRect(self):
return self.__outerRect
def paint(self, painter):
# connectors of a layer with the include.phase parameter
# have a different opacity than normal connectors
opacity = Constants.itemOpacityInPhase
if not self.__connector.getNodeEditor().isCurrentPhase(self.__connector.getPhase()):
opacity = Constants.itemOpacityNotInPhase
# get colors
colorInner = Constants.itemBackgroundColorLight
colorOuter = Constants.itemBackgroundColorDark
borderColor = Constants.itemBorderColor
# in place layer connectors have a different color than normal connectors
if self.__connector.isInPlace():
colorOuter = Constants.itemInPlaceColor
# set the colors opacity
colorInner.setAlpha(opacity)
colorOuter.setAlpha(opacity)
borderColor.setAlpha(opacity)
# draw enclosing rect and rect border
painter.setPen(QPen(borderColor, 1))
painter.fillRect(self.__outerRect, QBrush(colorOuter))
painter.drawRect(self.__outerRect)
# if the connector is connected, fill the inner rect
if self.__connector.isConnected():
painter.fillRect(self.__innerRect, QBrush(colorOuter))
else:
painter.fillRect(self.__innerRect, QBrush(colorInner))
# draw the border of the inner rect
painter.drawRect(self.__innerRect) Barista/gui/main_window/node_editor/items/renderer/node_item_renderer.py 0000664 ڊ0011117 00000031423 13265130053 027113 0 ustar klemms o0cvpr from PyQt5.QtCore import QRectF, Qt, QPointF
from PyQt5.QtGui import QPainter, QColor, QPen, QBrush, QLinearGradient
from gui.main_window.node_editor.node_editor_constants import Constants
from gui.main_window.node_editor.layer_color_definitions import LayerColorDefinitions
class TextRect:
""" Used to store values required for text positioning and rendering """
def __init__(self, text, rect, font = None):
self.text = text
self.rect = rect
self.font = font
class NodeItemRenderer:
""" Class is used to separate the drawing from the logic.
This class renders a given node to a graphics scene """
__blobConnectorSizeWithPadding = Constants.nodeItemConnectorPaddding * 2 + Constants.connectorItemOuterSize
def __init__(self, nodeItem):
self.__nodeItem = nodeItem
# create lists to store TextRects for the node item
self.__headRects = list()
self.__blobTopNameRects = list()
self.__blobBottomNameRects = list()
# create Qt rects for the head and body area
self.__rectHead = QRectF(0, 0, 1, 1)
self.__rectBlobArea = QRectF(0, 0, 1, 1)
# create Qt rect for enclosing rects used as bounding rects
self.__rectAll = QRectF(0, 0, 1, 1)
self.__rectAllSelected = QRectF(0, 0, 1, 1)
self.__hasCurrentPhase = True
self.update()
def boundingRect(self):
# selected layers have a outer 'glow', so selected layers are bigger than not selected layers
if self.__nodeItem.isSelected():
return self.__rectAllSelected
else:
return self.__rectAll
def update(self):
""" Recalculates positions of name, type, blobs, etc. """
# clear lists
del self.__headRects[:]
del self.__blobTopNameRects[:]
del self.__blobBottomNameRects[:]
# get size of the name
nameWidth = Constants.nodeItemFontNameMetrics.width(self.__nodeItem.getName()) + Constants.nodeItemHeadPadding
nameHeight = Constants.nodeItemFontTypeMetrics.height()
self.__headRects.append(TextRect(self.__nodeItem.getName(), QRectF(0, 0, nameWidth, nameHeight),
Constants.nodeItemFontName))
# get size of the type
typeWidth = Constants.nodeItemFontTypeMetrics.width(self.__nodeItem.getTypeString()) + Constants.nodeItemHeadPadding
typeHeight = Constants.nodeItemFontTypeMetrics.height()
self.__headRects.append(TextRect(self.__nodeItem.getTypeString(), QRectF(0, 0, typeWidth, typeHeight), Constants.nodeItemFontType))
# get size of the phase (if the node has the paramter include.phase
if len(self.__nodeItem.getPhase()) > 0:
phaseString = "Phase: " + self.__nodeItem.getPhase()
phaseWidth = Constants.nodeItemFontTypeMetrics.width(phaseString) + Constants.nodeItemHeadPadding
phaseHeight = Constants.nodeItemFontTypeMetrics.height()
self.__headRects.append(TextRect(phaseString, QRectF(0, 0, phaseWidth, phaseHeight),
Constants.nodeItemFontType))
# calculate rect width and height for names of blobs
topBlobsInfo = self.__buildBlobNameRectList(self.__nodeItem.getTopConnectors())
bottomBlobsInfo = self.__buildBlobNameRectList(self.__nodeItem.getBottomConnectors())
self.__blobTopNameRects = topBlobsInfo[0]
self.__blobBottomNameRects = bottomBlobsInfo[0]
blobAreaWidth = 2 * self.__blobConnectorSizeWithPadding + Constants.nodeItemConnectorPaddding + \
bottomBlobsInfo[1] + topBlobsInfo[1]
blobAreaHeight = max(topBlobsInfo[2], bottomBlobsInfo[2], Constants.nodeItemMinBlobAreaHeight)
# calculate the total width of the node
rectWidth = blobAreaWidth
for textRect in self.__headRects:
rectWidth = max(rectWidth, textRect.rect.width())
# change the width of the head rects to be the full width of the node
self.__updateHeadRects(rectWidth, self.__headRects)
# calculate the total node height
rectHeight = self.__rectHead.bottom() + blobAreaHeight
# create Qt rects for the body and enclosing width the top left at (0, 0)
self.__rectBlobArea = QRectF(0, self.__rectHead.bottom(), rectWidth, blobAreaHeight)
self.__rectAll = QRectF(0, 0, rectWidth, rectHeight)
# calculate a larger box to support selection (outer glow)
selSize = Constants.nodeItemSelectionSize
self.__rectAllSelected = QRectF(self.__rectAll.left() - selSize, self.__rectAll.top() - selSize,
self.__rectAll.width() + 2 * selSize, self.__rectAll.height() + 2 * selSize)
# after calculating the total width, adjust the name rects to align at the left/right
self.__adjustBlobNameRectPositions(self.__blobTopNameRects, self.__rectAll.width(), self.__rectHead.bottom(), False)
self.__adjustBlobNameRectPositions(self.__blobBottomNameRects, self.__rectAll.width(), self.__rectHead.bottom(), True)
def __updateHeadRects(self, finalWidth, headRects):
""" Resizes the size of the head text rect after the total needed width was calculated """
totalHeadHeight = 0
for x in range(0, len(headRects)):
# special case for the first head text -> start at (0, 0) and has space at the top
if x == 0:
headRects[x].rect = QRectF(0, 0, finalWidth, headRects[x].rect.height() + Constants.nodeItemTextMargin)
# special case for the last head text -> has space at the bottom
elif x == len(headRects) - 1:
headRects[x].rect = QRectF(0, headRects[x-1].rect.bottom(), finalWidth,
headRects[x].rect.height() + Constants.nodeItemTextMargin)
else:
headRects[x].rect = QRectF(0, headRects[x-1].rect.bottom(), finalWidth, headRects[x].rect.height())
# add the rects height to the total head rect height
totalHeadHeight += headRects[x].rect.height()
# create Qt rect for the head part
self.__rectHead = QRectF(0, 0, finalWidth, totalHeadHeight)
def __buildBlobNameRectList(self, connectors):
""" Creates rects for all top/bottom connectors (bounding rect for the text only) """
# check if the connector or the blob name text is higher
blobNameHeightRect = max(Constants.nodeItemFontBlobMetrics.height(), self.__blobConnectorSizeWithPadding)
blobNameRectList = list()
maxWidth = 0
height = 0
# for each connector calculate the width of the name and create a Qt rect.
# calculate the max width of all blob names and the total height
for item in connectors:
blobName = item.getBlobName()
blobNameWidth = Constants.nodeItemFontBlobMetrics.width(blobName) + Constants.nodeItemTextMargin
blobRect = QRectF(0, 0, blobNameWidth, blobNameHeightRect)
blobNameRectList.append(TextRect(blobName, blobRect))
maxWidth = max(maxWidth, blobNameWidth)
height += blobNameHeightRect
return blobNameRectList, maxWidth, height
def __adjustBlobNameRectPositions(self, blobRectNames, totalWidth, aboveHeight, atLeftBorder):
""" Recalculates the blob name rects after the total width of the node was calculated """
i = 0
for item in blobRectNames:
# position at the left side of the node
x = self.__blobConnectorSizeWithPadding
# or position at the right side of the node
if not atLeftBorder:
x = totalWidth - self.__blobConnectorSizeWithPadding - item.rect.width()
# calculate starting position (head size + position in body)
item.rect = QRectF(x, aboveHeight + i * item.rect.height(), item.rect.width(), item.rect.height())
i += 1
def updateConnectorPositions(self, connectors, atLeftBorder):
""" Repositions the connector items after the new size of the node was calculated """
i = 0
for item in connectors:
# position the connector item at the left border of the node,left of the blob name rect
x = Constants.nodeItemConnectorPaddding + Constants.connectorItemOuterSize / 2
# or at the right border of the node, right of the blob name rect
if not atLeftBorder:
x = self.__rectAll.width() - Constants.nodeItemConnectorPaddding - Constants.connectorItemOuterSize + \
Constants.connectorItemOuterSize / 2
# set the y coordinate of the connector item to be at the center of the blob name rect
y = self.__rectHead.bottom() + Constants.nodeItemConnectorPaddding + Constants.connectorItemOuterSize / 2 + \
i * self.__blobConnectorSizeWithPadding
item.setPos(QPointF(x, y))
i += 1
def paint(self, painter):
# use antialiasing for both text and box
painter.setRenderHint(QPainter.Antialiasing, True)
painter.setRenderHint(QPainter.TextAntialiasing, True)
# get the opacity of the node item, layers with the include.phase parameter have a different opacity
opacity = Constants.itemOpacityInPhase
if not self.__nodeItem.getNodeEditor().isCurrentPhase(self.__nodeItem.getPhase()):
opacity = Constants.itemOpacityNotInPhase
# draw a outer 'glow' around the node item if the node is selected
if self.__nodeItem.isSelected():
selectedColor = Constants.selectedColor
selectedColor.setAlpha(opacity)
painter.setPen(QPen(selectedColor, Constants.nodeItemSelectionSize))
painter.drawRect(self.__rectAllSelected)
painter.fillRect(self.__rectAllSelected, QBrush(selectedColor))
# get the type color for the header (name and type)
typeColor = LayerColorDefinitions.getTypeColor(self.__nodeItem.getType())
typeColor.setRgb(typeColor.red(), typeColor.green(), typeColor.blue(), opacity)
# get background color
backgroundColor = Constants.itemBackgroundColorLight
backgroundColor.setAlpha(opacity)
# set a linear gradient for the header (type color -> background color)
gradient = QLinearGradient(0, self.__rectHead.top(), 0, self.__rectHead.bottom())
gradient.setColorAt(0, typeColor)
gradient.setColorAt(0.5, backgroundColor)
# draw background and border for the header
painter.fillRect(self.__rectHead, QBrush(gradient))
# draw background for the blob area
painter.fillRect(self.__rectBlobArea, QBrush(backgroundColor))
borderColor = Constants.itemBorderColor
if self.__nodeItem.getIsInPlace():
borderColor = Constants.itemInPlaceColor
borderColor.setAlpha(opacity)
painter.setPen(QPen(borderColor, Constants.nodeItemBorderSize))
# draw outer border around the node
painter.drawRect(self.__rectAll)
# draw a line to separate header and connectors
borSize = Constants.nodeItemBorderSize
painter.setPen(QPen(borderColor, borSize))
painter.drawLine(self.__rectHead.left() + borSize / 2, self.__rectHead.bottom() - borSize / 2,
self.__rectHead.right() - borSize / 2, self.__rectHead.bottom() - borSize / 2)
painter.setPen(QPen(QColor(0, 0, 0, opacity)))
# draw text of header
if len(self.__headRects) > 1:
# align the first head text at the bottom to provide some space at the top
painter.setFont(self.__headRects[0].font)
painter.drawText(self.__headRects[0].rect, Qt.AlignHCenter | Qt.AlignBottom, self.__headRects[0].text)
# align other head texts at the center
for i in range(1, len(self.__headRects) - 1):
painter.setFont(self.__headRects[i].font)
painter.drawText(self.__headRects[i].rect, Qt.AlignCenter, self.__headRects[i].text)
# align the last head text at the top to provide some space at the bottom
painter.setFont(self.__headRects[-1].font)
painter.drawText(self.__headRects[-1].rect, Qt.AlignHCenter | Qt.AlignTop, self.__headRects[-1].text)
# there is only one head text, so align it at the center
elif len(self.__headRects) == 1:
painter.setFont(self.__headRects[0].font)
painter.drawText(self.__headRects[0].rect, Qt.AlignHCenter | Qt.AlignCenter, self.__headRects[0].text)
# draw blob names
painter.setFont(Constants.nodeItemFontBlob)
for item in self.__blobBottomNameRects:
painter.drawText(item.rect, Qt.AlignVCenter | Qt.AlignLeft, item.text)
for item in self.__blobTopNameRects:
painter.drawText(item.rect, Qt.AlignVCenter | Qt.AlignRight, item.text)
Barista/backend/barista/constraints/permanent/__init__.py 0000664 ڊ0011117 00000000506 13265130053 024052 0 ustar klemms o0cvpr """
This subpackage contains all constraints that need to be ensured permanently.
So the given methods need to be called each time one of the handled constraints might get broken, which is usually
identically to all situations in which the user could edit the raw prototxt file or equivalent data (as in the project
file).
""" Barista/backend/barista/constraints/permanent/project.py 0000664 ڊ0011117 00000002350 13265130053 023760 0 ustar klemms o0cvpr """
This module defines permanent constraints for a Barista project.
"""
from backend.barista.utils.logger import Log
from backend.barista.constraints.permanent.solver import ensureSolverConstraints
logID = Log.getCallerId('constraints/project')
def ensureProjectDataConstraints(projectData):
"""Take the current state of projectData and manipulate it to handle some special cases.
This method should be called each time when the project data has been changed. It will adjust some values, which
would be valid for the prototxt-syntax in general, but underlay further constraints especially for Barista.
"""
if not hasattr(projectData, '__getItem__'):
Log.error('Project data is empty.', logID)
return projectData
jsonKeys = ("activeSession", "environment", "inputdb", "projectid", "transform")
keyNotFound = False
for key in jsonKeys:
if key not in projectData:
Log.error("Project information is invalid! Key '"
+ key + "' is missing!", logID)
keyNotFound = True
if keyNotFound:
Log.error("Session at %s is invalid. Key %s in 'sessionstate.json' is missing", logID)
# add further constraint types like e.g.:
return projectData
Barista/backend/barista/constraints/permanent/solver.py 0000664 ڊ0011117 00000003641 13265130053 023630 0 ustar klemms o0cvpr """
This module defines permanent solver constraints.
"""
import os
import backend
from backend.barista.utils.logger import Log
logID = Log.getCallerId('constraints/solver')
def ensureSolverConstraints(solverDictionary):
"""Ensure that all constraints for the given solverDictionary are valid.
Sets static values and removes invalid values.
"""
# The file names inside of a session are static and must not be altered by the user
if "net" not in solverDictionary or solverDictionary["net"] != backend.barista.session.session_utils.Paths.FILE_NAME_NET_INTERNAL:
Log.log("The solver property 'net' must point to the generated network file. "
"Value has been changed from '{}' to '{}'.".format(
solverDictionary["net"] if "net" in solverDictionary else "None",
backend.barista.session.session_utils.Paths.FILE_NAME_NET_INTERNAL
), logID)
solverDictionary["net"] = backend.barista.session.session_utils.Paths.FILE_NAME_NET_INTERNAL
# An additional net definition inside of the solver would be inconsistent to the separately handled network
if "net_param" in solverDictionary:
Log.log("The solver property 'net_param' is not supported as it would be inconsistent with the separately "
"handled network. Property has been removed.", logID)
del solverDictionary["net_param"]
# a snapshot_prefix containing a path is not supported either
if "snapshot_prefix" in solverDictionary:
head, tail = os.path.split(solverDictionary["snapshot_prefix"])
if len(head) > 0:
Log.log("The solver property 'snapshot_prefix' contained an unsupported path. "
"Property was shortened from '{}' to '{}'.".format(
solverDictionary["snapshot_prefix"],
tail
), logID)
solverDictionary["snapshot_prefix"] = tail
return solverDictionary
Barista/backend/barista/constraints/session_run/__init__.py 0000664 ڊ0011117 00000000300 13265130053 024420 0 ustar klemms o0cvpr """
This subpackage contains all constraints that need to be ensured before/during a new session run.
So the given methods need to be called each time a new session is about to be started.
""" Barista/backend/barista/constraints/session_run/training.py 0000664 ڊ0011117 00000060563 13265130053 024515 0 ustar klemms o0cvpr """
This subpackage contains all constraints that need to be ensured before/during the training only.
So the given methods need to be called each time a new training session is about to be started.
"""
import os
from PyQt5 import QtWidgets
from backend.barista.constraints import common
from backend.caffe.proto_info import CaffeMetaInformation
from gui.network_manager.layer_helper import LayerHelper
from backend.caffe.proto_info import UnknownLayerTypeException
def checkMinimumTrainingRequirements(session, parentGui=None, reportToUser=True):
""" Check whether the minimum requirements for starting a new training session are met.
This method should be called whenever a new training session is about to be started. If the returned value isn't
True, the session start should be aborted.
TODO implement the same for a evaluation-only net.
If reportToUser is set to True, which is the default value, the user will be informed about unmet requirements by a
message box, including an option to ignore all listed errors.
:return: True, if everything is okay. False otherwise.
"""
# validate
requirements = MinimumTrainingRequirements(session)
valid = requirements.valid()
session.setErrorList(requirements.getErrors())
# list errors and let the user decide whether to ignore the errors
if (parentGui is not None):
if reportToUser and not valid:
valid = requirements.showErrors(parentGui)
return valid
else: # if there is no GUI, return the Error Messages
return requirements.getErrors()
class MinimumTrainingRequirements:
"""This class is only required inside of checkMinimumTrainingRequirements.
Do not use as a public interface.
"""
def __init__(self, session):
self._session = session
self._stateData = None
if session.state_dictionary:
self._stateData = session.state_dictionary
# store one error message per unmet requirement in the following list
self._errorMessages = []
# start validation
self._check()
def showErrors(self, parentGui):
""" If any errors have been collected, show them in a message box.
Additionally, to the listed errors, the user will get an option to ignore all errors.
:param parentGui: The parent GUI to which the shown message box will be bounded.
:return: Return true, iff no errors occured or the user decided to ignore them.
"""
valid = self.valid()
if not valid:
# accumulate collected error data
msg = ""
for msg_part in self._errorMessages:
if len(msg) > 0:
msg += "\n\n"
msg += msg_part
# configure message box
msgBox = QtWidgets.QMessageBox(parentGui)
msgBox.setIcon(QtWidgets.QMessageBox.Warning)
msgBox.setText(parentGui.tr("Please check the following constraints.\nIf you decide to ignore this message, "
"it might result in a program crash."))
msgBox.setInformativeText(parentGui.tr(msg))
msgBox.setWindowTitle(parentGui.tr("Minimum training requirements aren't met."))
msgBox.setParent(parentGui)
msgBox.setStandardButtons(QtWidgets.QMessageBox.Ignore | QtWidgets.QMessageBox.Ok)
msgBox.setDefaultButton(QtWidgets.QMessageBox.Ok)
# show message box and receive user input
clickedButton = msgBox.exec_()
# if the user clicked "ignore", all errors will be ignored
valid = clickedButton == QtWidgets.QMessageBox.Ignore
return valid
def valid(self):
"""Check whether any constraint has been broken."""
return len(self._errorMessages) == 0
def getErrors(self):
return self._errorMessages
def _check(self):
"""Start all validations."""
# TODO check type(self._session) for
# temporary change the current working dir to allow evaluation of relative paths
if hasattr(self._session, 'checkTraining'):
self._errorMessages.extend(self._session.checkTraining())
elif self._checkLayers():
self._checkSolver()
self._checkDataLayerExistence()
self._checkDataLayerParameters()
self._checkInputLayer()
self._checkUniqueBlobNames()
def _checkLayers(self):
"""Checks ify the network contains layers that are not compatible with the current caffe-version"""
if self._stateData is not None:
layers = []
for layer in self._stateData["network"]["layers"]:
layers.append(self._stateData["network"]["layers"][layer]["parameters"]["type"])
try:
for layer in layers:
typename = CaffeMetaInformation().getLayerType(layer)
except UnknownLayerTypeException as e:
self._errorMessages.append((e._msg, "Unknown Layer"))
return False
return True
def _checkSolver(self):
"""Check whether all solver constraints are valid."""
# the base learning rate should be a positive number
if not hasattr(self._stateData, '__getitem__'):
self._errorMessages.append(("State data is empty!","..."))
return
if self._stateData["solver"] == {u'net': u'net-internal.prototxt'}:
self._errorMessages.append(("The solver seems to be empty. Please import or define a solver.\n"
"(You can define a solver with the help of the dock Solver Properties)","No solver."))
if not ("base_lr" in self._stateData["solver"]
and common.isPositiveNumber(self._stateData["solver"]["base_lr"])):
self._errorMessages.append(("The base learning rate (base_lr) must be a positive number.","Invalid base_lr."))
# the maximum number of iterations should be a positive integer
if not ("max_iter" in self._stateData["solver"]
and common.isPositiveInteger(self._stateData["solver"]["max_iter"])):
self._errorMessages.append(("The maximum number of iterations (max_iter) must be a positive integer.","Invalid max_iter."))
def _checkUniqueBlobNames(self):
"""Checks for duplicate top blob names in all layers and emits error message if duplicates found except
if in-place is permitted"""
# find all 'real' sources of blobs, i.e. layers that produce a blob as output and are not in-place
blobGenerators = {} # dictionary that saves the sources for every blob name
if hasattr(self._stateData, '__getitem__'):
for layer_id, layer in self._stateData["network"]["layers"].iteritems():
parameters = layer.get("parameters", {})
tops = parameters.get("top", [])
bottoms = parameters.get("bottom", [])
# if at least one top blob is also a bottom, check if layer allows in-place
sourced = [blob for blob in tops if blob not in bottoms]
inPlace = [blob for blob in tops if blob in bottoms]
if len(inPlace) > 0 and not layer["type"].allowsInPlace():
for name in inPlace:
self._errorMessages.append((
"{} is reproduced by {}".format(name, parameters.get("name", "[NO NAME]")),
"{} does not support in-place operation".format(parameters.get("name",
"[NO NAME]"))
))
# check all blobs that are generated in one layer if they are generated only once in each phase.
for name in sourced:
phase = [] # this list can hold train, test and both
p = LayerHelper.getLayerPhase(layer)
if p == "":
phase = [LayerHelper.PHASE_TEST, LayerHelper.PHASE_TRAIN]
else:
phase.append(p)
# if a blob already exists, check if it was generated before in the same Phase
if name in blobGenerators:
found_match = False
for candidate in blobGenerators[name]:
intersection = set(phase).intersection(candidate[1])
if len(intersection) > 0:
found_match = True
self._errorMessages.append(("Sources are {} and {} in phase {}".format(
parameters.get("name", "[NO NAME]"),
candidate[0],
list(intersection)[0]),
"{} is generated by multiple layers".format(name)))
if not found_match:
blobGenerators[name].append((parameters.get("name", "[NO NAME]"), phase))
else:
blobGenerators[name] = [(parameters.get("name", "[NO NAME]"), phase)]
def _checkDataLayerExistence(self):
"""Check whether at least one data layer exists (during the training phase)."""
nDataLayerTotal = 0
nDataLayerTraining = 0
if not hasattr(self._stateData, '__getitem__'):
self._errorMessages.append(("State data is empty!","Empty state data."))
return
if self._stateData["network"]["layers"] == {}:
self._errorMessages.append(("There is no net defined. Please import or create a network.","No net."))
try:
for layer_id, layer in self._stateData["network"]["layers"].iteritems():
if layer["type"].isDataLayer():
nDataLayerTotal += 1
if LayerHelper.isLayerIncludedInTrainingPhase(layer):
nDataLayerTraining += 1
except KeyError:
self._errorMessages.append(("No network data available.","No network data."))
# list all available data layers to inform the user about possible options
typeListMsg = ""
allTypes = CaffeMetaInformation().availableLayerTypes()
for key in sorted(allTypes):
if allTypes[key].isDataLayer():
if typeListMsg != "":
typeListMsg += ", "
typeListMsg += key
typeListMsg = " Available types of such a layer are: " + typeListMsg + "."
if nDataLayerTotal == 0:
self._errorMessages.append(("There should be at least one data layer in your network.\n" + typeListMsg,"No/invalid input data."))
elif nDataLayerTraining == 0:
self._errorMessages.append(("There should be at least one data layer in your network that is included in "
"the training phase.\n" + typeListMsg,"No/invalid input data."))
def _checkDataLayerParameters(self):
"""Check whether the existing data layers (of the training phase) provide (valid) data.
required parameters were determined based on http://caffe.berkeleyvision.org/tutorial/layers.html
"""
if not hasattr(self._stateData, '__getitem__'):
self._errorMessages.append(("State data is empty!","Empty state data."))
return
# a list of pairs. first element of a pair is the name of a layer (type).
# The second one is the name of the missing parameter.
self._missingLayerParams = []
for layer_id, layer in self._stateData["network"]["layers"].iteritems():
if layer["type"].isDataLayer() and LayerHelper.isLayerIncludedInTrainingPhase(layer):
if layer["type"].name() == "Data":
self._checkDataLayer(layer)
elif layer["type"].name() == "MemoryData":
self._checkMemoryDataLayer(layer)
elif layer["type"].name() == "HDF5Data":
self._checkHDF5DataLayer(layer)
elif layer["type"].name() == "ImageData":
self._checkImageDataLayer(layer)
elif layer["type"].name() == "WindowData":
self._checkWindowDataLayer(layer)
elif layer["type"].name() == "DummyData":
self._checkDummyDataLayer(layer)
for pair in self._missingLayerParams:
self._errorMessages.append(("The {} layer must provide the {} parameter.".format(
pair[0],
pair[1]
),"Missing parameter."))
def _checkDataLayer(self, layer):
"""Validate a layer of type Data."""
specificParamKey = "data_param"
if specificParamKey not in layer["parameters"]:
self._missingLayerParams.append([layer["type"].name(), specificParamKey])
else:
specificParams = layer["parameters"][specificParamKey]
# source
if "source" not in specificParams:
self._missingLayerParams.append([layer["type"].name(), specificParamKey + ".source"])
elif not os.path.isdir(self._makeAbsPath(specificParams["source"])):
self._errorMessages.append((
"The parameter {} of the layer {}\ndoes not seem to point to a valid directory. \n"
"Please import and connect databases (via Input Manager).".format(specificParamKey + ".source", layer["type"].name()),
"No/invalid input data."))
# TODO check whether the given directory contains files that fit to the given database type.
# batch_size
if "batch_size" not in specificParams:
self._missingLayerParams.append([layer["type"].name(), specificParamKey + ".batch_size"])
elif not common.isPositiveInteger(specificParams["batch_size"]):
self._errorMessages.append(("The batch size (of a {} Layer) must be a positive integer.".format(
layer["type"].name()
),"Invalid batch size."))
def _checkMemoryDataLayer(self, layer):
"""Validate a layer of type MemoryData."""
specificParamKey = "memory_data_param"
if specificParamKey not in layer["parameters"]:
self._missingLayerParams.append([layer["type"].name(), specificParamKey])
else:
specificParams = layer["parameters"][specificParamKey]
# batch_size
if "batch_size" not in specificParams:
self._missingLayerParams.append([layer["type"].name(), specificParamKey + ".batch_size"])
elif not common.isPositiveInteger(specificParams["batch_size"]):
self._errorMessages.append(("The batch size (of a {} Layer) must be a positive integer.".format(
layer["type"].name()
),"Invalid batch size."))
# channels
if "channels" not in specificParams:
self._missingLayerParams.append([layer["type"].name(), specificParamKey + ".channels"])
elif not common.isPositiveInteger(specificParams["channels"]):
self._errorMessages.append(("The number of channels (of a {} Layer) must be a positive integer.".format(
layer["type"].name()
),"Invalid number of channels."))
# height
if "height" not in specificParams:
self._missingLayerParams.append([layer["type"].name(), specificParamKey + ".height"])
elif not common.isPositiveInteger(specificParams["height"]):
self._errorMessages.append(("The height parameter (of a {} Layer) must be a positive integer.".format(
layer["type"].name()
),"Invalid height parameter."))
# width
if "width" not in specificParams:
self._missingLayerParams.append([layer["type"].name(), specificParamKey + ".width"])
elif not common.isPositiveInteger(specificParams["width"]):
self._errorMessages.append(("The width parameter (of a {} Layer) must be a positive integer.".format(
layer["type"].name()
),"Invalid width parameter."))
# actually, this layer isn't supported in the training at all. however, the above validation is kept to
# allow using Barista to build(!) a valid net definition without(!) the possibility to start training.
self._errorMessages.append(('Layers of type "MemoryData" aren\'t supported yet, as the in-memory data\n'
'reference needs to be specified in the code. So you can\'t run a Barista\n'
'session right now. However, you can use Barista to\n'
'build and export a valid net definition containing this layer type.',"..."))
def _checkHDF5DataLayer(self, layer):
"""Validate a layer of type HDF5Data."""
specificParamKey = "hdf5_data_param"
if specificParamKey not in layer["parameters"]:
self._missingLayerParams.append([layer["type"].name(), specificParamKey])
else:
specificParams = layer["parameters"][specificParamKey]
# source
if "source" not in specificParams:
self._missingLayerParams.append([layer["type"].name(), specificParamKey + ".source"])
elif not os.path.isfile(self._makeAbsPath(specificParams["source"])):
self._errorMessages.append((
"The parameter {} of the layer {}\ndoes not seem to point to a valid "
"file.".format(specificParamKey + ".source", layer["type"].name()),"No/invalid input data."))
# TODO check whether the given file is really a (valid) HDF5 file.
# batch_size
if "batch_size" not in specificParams:
self._missingLayerParams.append([layer["type"].name(), specificParamKey + ".batch_size"])
elif not common.isPositiveInteger(specificParams["batch_size"]):
self._errorMessages.append(("The batch size (of a {} Layer) must be a positive integer.".format(
layer["type"].name()
),"Invalid batch size."))
def _checkImageDataLayer(self, layer):
"""Validate a layer of type ImageData."""
specificParamKey = "image_data_param"
if specificParamKey not in layer["parameters"]:
self._missingLayerParams.append([layer["type"].name(), specificParamKey])
else:
specificParams = layer["parameters"][specificParamKey]
# source
if "source" not in specificParams:
self._missingLayerParams.append([layer["type"].name(), specificParamKey + ".source"])
elif not os.path.isfile(self._makeAbsPath(specificParams["source"])):
self._errorMessages.append((
"The parameter {} of the layer {}\ndoes not seem to point to a valid "
"file.".format(specificParamKey + ".source", layer["type"].name()),"No/invalid input data."))
# TODO check whether the given file is really a (valid) text file that contains (valid) image paths.
# batch_size
if "batch_size" not in specificParams:
self._missingLayerParams.append([layer["type"].name(), specificParamKey + ".batch_size"])
elif not common.isPositiveInteger(specificParams["batch_size"]):
self._errorMessages.append(("The batch size (of a {} Layer) must be a positive integer.".format(
layer["type"].name()
),"Invalid batch size."))
def _checkWindowDataLayer(self, layer):
"""Validate a layer of type WindowData.
Constraints are described here:
http://caffe.berkeleyvision.org/doxygen/classcaffe_1_1WindowDataLayer.html#details
"""
specificParamKey = "window_data_param"
if specificParamKey not in layer["parameters"]:
self._missingLayerParams.append([layer["type"].name(), specificParamKey])
else:
specificParams = layer["parameters"][specificParamKey]
# source
if "source" not in specificParams:
self._missingLayerParams.append([layer["type"].name(), specificParamKey + ".source"])
elif not os.path.isfile(self._makeAbsPath(specificParams["source"])):
self._errorMessages.append((
"The parameter {} of the layer {}\ndoes not seem to point to a valid "
"file.".format(specificParamKey + ".source", layer["type"].name()),"No/invalid input data."))
# TODO add further validation checks for this type (problem: documentation lacks a lot).
# batch_size
if "batch_size" not in specificParams:
self._missingLayerParams.append([layer["type"].name(), specificParamKey + ".batch_size"])
elif not common.isPositiveInteger(specificParams["batch_size"]):
self._errorMessages.append(("The batch size (of a {} Layer) must be a positive integer.".format(
layer["type"].name()
),"Invalid batch size."))
def _checkDummyDataLayer(self, layer):
"""Validate a layer of type DummyData.
Constraints are described in the comments of the caffe.proto file.
"""
specificParamKey = "dummy_data_param"
if specificParamKey not in layer["parameters"]:
self._missingLayerParams.append([layer["type"].name(), specificParamKey])
else:
specificParams = layer["parameters"][specificParamKey]
# shape
if "shape" not in specificParams:
self._missingLayerParams.append([layer["type"].name(), specificParamKey + ".shape"])
elif not len(specificParams["shape"]) >= 1:
self._errorMessages.append((
"The parameter {} of the layer {}\nneeds at least one value.".format(
specificParamKey + ".shape", layer["type"].name()
),"Missing parameter."))
def _checkInputLayer(self):
"""Check for existence of an input layer.
Note that, this is about the specific layer type called "Input". This is not about the general group of Data
layers (see tests above).
"""
if not hasattr(self._stateData, '__getitem__'):
self._errorMessages.append(("State data is empty!","Empty state data."))
return
for layer_id, layer in self._stateData["network"]["layers"].iteritems():
if layer["type"].name() == "Input":
self._errorMessages.append(('Layers of type "Input" aren\'t supported yet, as they are usually not meant\n'
'to be included before deploying the net. So you can\'t run a Barista\n'
'session right now. However, you can use Barista to build and export a valid\n'
'net definition containing this layer type.',"..."))
break
def _makeAbsPath(self, path):
# TODO: remove the print once errorMessages are shown to the user
if os.path.isabs(path):
return os.path.normpath(path)
else: # if path is a relative path, see if it is relative to the session folder
sessionDir = self._session.getDirectory()
offsetDir = os.path.normpath(os.path.abspath(sessionDir))
candidate = os.path.normpath(os.path.join(offsetDir, path))
lastCandidate = ''
while (not os.path.exists(candidate)) and not (candidate == lastCandidate):
lastCandidate = candidate
offsetDir = os.path.normpath(os.path.join(offsetDir, os.pardir))
candidate = os.path.normpath(os.path.join(offsetDir, path))
if os.path.exists(candidate):
return candidate
else:
print("Could not create an absolute path from: {} and sessionDir {}".format(path, sessionDir))
return path
# TODO: remove if not called from anywhere else
def _makeSessionPath(self, path):
raise NotImplementedError
"""add ../../ to path to move from project to session level"""
for i in range(0, self.sessionDirDepth):
path = os.path.join(os.pardir, path)
return path
Barista/gui/main_window/docks/plotter/__init__.py 0000664 ڊ0011117 00000000000 13265130053 022356 0 ustar klemms o0cvpr Barista/gui/main_window/docks/plotter/plot_canvas.py 0000664 ڊ0011117 00000004025 13265130053 023143 0 ustar klemms o0cvpr
from PyQt5.QtWidgets import QSizePolicy
from matplotlib.backends.backend_qt5agg import FigureCanvasQTAgg as FigureCanvas
from matplotlib.figure import Figure
def handleNonPositives(yValues):
allNonPositive = []
for i in range(0,len(yValues)):
for j in range(0,len(yValues[i])):
if len(allNonPositive) < j + 1:
allNonPositive.append(True)
if yValues[i][j] > 0:
allNonPositive[j] = False
for i in range(0,len(yValues)):
yValues[i] = [yValues[i][j] for j in range(0,len(allNonPositive)) if not allNonPositive[j]]
return yValues
def allNonNegative(yValues):
for ys in yValues:
for y in ys:
if y > 0:
return False
return True
class PlotCanvas(FigureCanvas):
"""
This plotter canvas draws the actual plots within the plotter widget.
"""
def __init__(self, parent=None, width=5, height=4, dpi=100):
self.fig = Figure(figsize=(width, height), dpi=dpi)
self.axes = self.fig.add_subplot(111)
#self.axes.hold(True) # MatPlotLib hold depricated http://matplotlib.org/api/api_changes.html
self.axes.grid(True)
FigureCanvas.__init__(self, self.fig)
self.setParent(parent)
FigureCanvas.setSizePolicy(self,
QSizePolicy.Expanding,
QSizePolicy.Expanding)
FigureCanvas.updateGeometry(self)
def plotDictList(self, xValues, yValues, logarithmic):
"""
Plot a multiple charts on the axes.
Return the Line2D docks.
"""
self.axes.grid(True)
if logarithmic and not allNonNegative(yValues):
try:
return self.axes.semilogy(xValues, handleNonPositives(yValues))
except:
pass
return self.axes.plot(xValues, yValues)
def legend(self, lines, labels):
self.axes.legend(lines, labels, loc = 0, fontsize=10)
def clear(self):
self.axes.clear()
self.axes.grid(True)
Barista/gui/main_window/docks/plotter/qt5_plotter.py 0000664 ڊ0011117 00000034235 13265130053 023122 0 ustar klemms o0cvpr import csv
import time
import seaborn
from collections import OrderedDict
from PyQt5 import QtCore
from PyQt5.QtWidgets import QVBoxLayout, QWidget, QFrame
from backend.parser.parser_listener import ParserListener
from backend.parser.parser import Parser
from backend.parser.parser_dummy import ParserDummy
from gui.main_window.docks.plotter.plot_canvas import PlotCanvas
# Own class for plotter
# Select Train/Test
# Select one or more value types: training rate, acc0, acc1, loss
# Plot against time or iteration
# Label the coordinates
# More features? Multiple Plots...
class Qt5Plotter(QFrame):
"""
This plotter takes a list of dictionaries as input and plottes
selected data
"""
# available metrics
global TIME, ITERATION, LEARNING_RATE
TIME = "Seconds"
ITERATION = "NumIters"
LEARNING_RATE = "LearningRate"
# How will be plottet? Time <-> Iterations, Linear <-> Logarithmic
__againstTime = False
__logarithmic = False
# Each parser (i.e. log) has a list of metrics of train and test values
# that the user wants to be plotted
# Each list entry is a set of the metric that will be plotted
# logId -> {metric}
__testMetrics = OrderedDict()
__trainMetrics = OrderedDict()
# Each parser (i.e. log) has a list of train and test data
# Each list entry is again a dictionary of the parsed key-value-pairs
# logId -> [key -> value]
__trainData = OrderedDict()
__testData = OrderedDict()
# Logs can be plotted with an offset in time/iterations
# This is a usefull parameter for concatenating logs
__timeOffsets = OrderedDict()
__iterationOffsets = OrderedDict()
# The list of parsers corresponds to the list of logs that are observed.
# Each log (file or stream) must be added using a parser
# logId -> parser
__parsers = OrderedDict()
__listener = OrderedDict()
__plotterGUI = None
class ParserConnection(ParserListener):
"""
Implements the required methods for the ParserListener so that the
parsed data can be added to the data that will be plotted.
"""
def __init__(self, logId, testData, trainData, plotter, plotOnUpdate=False):
"""
Register the test and train data lists of the plotter. Optionaly
set plotOnUpdate to True to replot the data every time the parser
reads a new row and triggers an update.
"""
self.logId = logId
self.plotter = plotter
self.testData = testData
self.trainData = trainData
self.plotOnUpdate = plotOnUpdate
def update(self, phase, row):
"""
Pass the update signal to the plotter. Replot if that is wanted.
"""
if phase == "TEST":
self.testData.append(row)
if phase == "TRAIN":
self.trainData.append(row)
if self.plotOnUpdate:
self.plotter.plot(False, [self.logId])
def handle(self, event, message, groups):
# Ignore event.
# Not the plotters business.
pass
def registerKey(self, phase, key):
"""
Pass the signal to the gui so that it can create a new checkbox for the key.
"""
gui = self.plotter.getPlotterGUI()
if gui:
gui.registerKeySignal.emit(self.logId, phase, key)
# end ParserConnection
def __init__(self, parent=None):
super(Qt5Plotter, self).__init__(parent)
self.setFrameShape(QFrame.StyledPanel)
# this is the Canvas Widget that displays the `figure`
self.canvas = PlotCanvas(self)
# this is the Navigation widget
# it takes the Canvas widget and a parent
# self.toolbar = NavigationToolbar(self.canvas, self)
# set the layout
layout = QVBoxLayout()
layout.setContentsMargins(0,0,0,0)
# layout.addWidget(self.toolbar)
layout.addWidget(self.canvas)
self.setLayout(layout)
def resizeEvent(self, event):
# This keeps the plot area as large as possible even after resizing the widget
try:
self.canvas.fig.tight_layout()
except:
pass
def setPlotterGUI(self, plotterGUI):
self.__plotterGUI = plotterGUI
def getPlotterGUI(self):
return self.__plotterGUI
def getParser(self, logId):
"""
Get the parser that corresponds to the given log name.
"""
return self.__parsers[logId][1]
def putLog(self, logId, logName, parser, plotOnUpdate=False, timeOffset=0, iterationOffset=0):
"""
Add a new log to be plotted. A log is added by adding a parser that
reads the log file/stream. A logId is used to identify the
logs/parsers
It is possible to add offset values for time and iteration numbers
This is usually only interesting for the time, e.g. if a training is
interruped and resumed (usually the iteration number in the log-file
is still constistent.
"""
if (isinstance(parser, Parser) | isinstance(parser, ParserDummy)) & isinstance(logId, str):
self.__testData[logId] = []
self.__trainData[logId] = []
self.__testMetrics[logId] = []
self.__trainMetrics[logId] = []
self.__timeOffsets[logId] = timeOffset
self.__iterationOffsets[logId] = iterationOffset
self.__listener[logId] = self.ParserConnection(
logId,
self.__testData[logId],
self.__trainData[logId],
self,
plotOnUpdate)
parser.addListener(self.__listener[logId])
self.__parsers[logId] = [logName, parser]
def removeLog(self, logId):
"""
Remove the given log from the plot.
"""
if logId:
if logId in self.__parsers:
if logId in self.__listener:
self.__parsers[logId][1].removeListener(self.__listener[logId])
self.__parsers.pop(logId)
def showMetricTest(self, logId, metric, show=True):
"""
Add a metric to the test plot if show is true, otherwise remove
the metric. logId is the name of the logfile which is plotted.
"""
if logId not in self.__testMetrics:
return
if show and metric not in self.__testMetrics[logId]:
self.__testMetrics[logId].append(metric)
elif not show and metric in self.__testMetrics[logId]:
self.__testMetrics[logId].remove(metric)
def isMetricTestShown(self, logId, metric):
return metric in self.__testMetrics[logId]
def showMetricTrain(self, logId, metric, show=True):
"""
Add a metric to the train plot if show is true, otherwise remove
the metric. logId is the name of the logfile which is plotted.
"""
if logId not in self.__testMetrics:
return
if show and metric not in self.__trainMetrics[logId]:
self.__trainMetrics[logId].append(metric)
elif not show and metric in self.__trainMetrics[logId]:
self.__trainMetrics[logId].remove(metric)
def isMetricTrainShown(self, logId, metric):
return metric in self.__trainMetrics[logId]
def plotAgainstTime(self):
self.__againstTime = True
def plotAgainstIterations(self):
self.__againstTime = False
def plotLogarithmic(self):
self.__logarithmic = True
def plotLinear(self):
self.__logarithmic = False
#
# Some helping methodes for the plotting # # # # # # # # # # # #
#
def __xLabel(self):
if self.__againstTime:
return "Time (in s)"
else:
return "Iterations"
def __xValues(self, dictList, logId, againstTime):
xValues = []
if againstTime:
for dict in dictList:
xValues.append(dict.get(TIME) + self.__timeOffsets[logId])
else:
for dict in dictList:
xValues.append(dict.get(ITERATION) +
self.__iterationOffsets[logId])
return xValues
def __yValues(self, dictList, showMetrics):
yValues = []
for record in dictList:
values = []
for metric in showMetrics:
values.append(record.get(metric))
yValues.append(values)
return yValues
# Count the metrics the user wants to plot
def numGraphs(self):
numGraphs = 0
for logId in self.__parsers:
numGraphs = numGraphs + len(self.__trainMetrics[logId]) + len(self.__testMetrics[logId])
return numGraphs
def plot(self, evenIfThereIsNothingToPlot = True, updateOnly = None):
"""
Plot the data. This method will don't call the final plot function of the actual
plotting library if there is no data selected for plotting. Otherwise there could
result an unpretty visual flickering in the widget.
One can plot anyway by setting the first parameter to False. In addition one can
further specify which logs should be checked in order to avoid unnecessary replotting.
"""
# Count the metrics the user wants to plot
# Only plot it if the number is greater than 0 or if the user wants to plot the empty plot
numGraphs = 0
relevantParsers = self.__parsers
if updateOnly is not None:
relevantParsers = updateOnly
for logId in relevantParsers:
numGraphs = numGraphs + len(self.__trainMetrics[logId]) + len(self.__testMetrics[logId])
if numGraphs == 0 and not evenIfThereIsNothingToPlot:
return
# Begin plotting
self.canvas.clear()
lines = []
labels = []
for logId in self.__parsers:
if len(self.__trainMetrics[logId]) != 0:
xValues = self.__xValues(self.__trainData[logId], logId, self.__againstTime)
yValues = self.__yValues(self.__trainData[logId],
self.__trainMetrics[logId])
lines.extend(self.canvas.plotDictList(xValues, yValues, self.__logarithmic))
train = lambda s: self.__parsers[logId][0] + ".train." + s
labels.extend(map(train, self.__trainMetrics[logId]))
if len(self.__testMetrics[logId]) != 0:
xValues = self.__xValues(self.__testData[logId], logId, self.__againstTime)
yValues = self.__yValues(self.__testData[logId],
self.__testMetrics[logId])
lines.extend(self.canvas.plotDictList(xValues, yValues, self.__logarithmic))
test = lambda s: self.__parsers[logId][0] + ".test." + s
labels.extend(map(test, self.__testMetrics[logId]))
self.canvas.legend(lines, labels)
try:
self.canvas.draw()
except:
pass
def getLastTimeValue(self, logId, ofTestData=False):
if ofTestData:
return self.__testData[logId][-1][TIME]
else:
return self.__trainData[logId][-1][TIME]
def getLastIterationValue(self, logId, ofTestData=False):
if ofTestData:
return self.__testData[logId][-1][ITERATION]
else:
return self.__trainData[logId][-1][ITERATION]
# Methods for CSV export # # # # # # # # # # # # # # # # # # # # # # #
def __arrayToString(self, array, delimiter):
if array is None or len(array) == 0:
return ""
else:
string = array[0]
for i in range(1, len(array)):
string = string + delimiter + " " + array[i]
return string
def exportCSVToFile(self, path):
if not path:
return
with open(path, 'wb') as csvfile:
spamwriter = csv.writer(csvfile, delimiter=';',
quotechar='|', quoting=csv.QUOTE_MINIMAL)
spamwriter.writerow(["# Barista CSV-Export (" + time.ctime() + ")"])
for logId in self.__parsers:
names = []
if len(self.__trainMetrics[logId]) != 0:
xIterValues = self.__xValues(self.__trainData[logId], logId, False)
xTimeValues = self.__xValues(self.__trainData[logId], logId, True)
yValues = self.__yValues(self.__trainData[logId],
self.__trainMetrics[logId])
train = lambda s: self.__parsers[logId][0] + ".train." + s
names = map(train, self.__trainMetrics[logId])
spamwriter.writerow([])
spamwriter.writerow([])
spamwriter.writerow(["# " + self.__parsers[logId][0] + " TRAIN"])
row = ["Iterations", "Time"]
row.extend(names)
spamwriter.writerow(row)
for i in range(0, len(xIterValues)):
row = [xIterValues[i], xTimeValues[i]]
row.extend(yValues[i])
spamwriter.writerow(row)
if len(self.__testMetrics[logId]) != 0:
xIterValues = self.__xValues(self.__testData[logId], logId, False)
xTimeValues = self.__xValues(self.__testData[logId], logId, True)
yValues = self.__yValues(self.__testData[logId],
self.__testMetrics[logId])
test = lambda s: self.__parsers[logId][0] + ".test." + s
spamwriter.writerow([])
spamwriter.writerow([])
spamwriter.writerow(["# " + self.__parsers[logId][0] + " TEST"])
names = map(test, self.__testMetrics[logId])
row = ["Iterations", "Time"]
row.extend(names)
spamwriter.writerow(row)
for i in range(0, len(xIterValues)):
row = [int(xIterValues[i]), xTimeValues[i]]
row.extend(yValues[i])
spamwriter.writerow(row)
Barista/gui/main_window/docks/plotter/qt5_plotter_gui.py 0000664 ڊ0011117 00000053165 13265130053 023771 0 ustar klemms o0cvpr import os
from PyQt5 import QtGui, QtCore, QtWidgets
from collections import OrderedDict
from PyQt5.QtCore import Qt, pyqtSignal, QVariant
from PyQt5.QtWidgets import (
QComboBox,
QHBoxLayout,
QVBoxLayout,
QGridLayout,
QPushButton,
QCheckBox,
QListWidget,
QFileDialog,
QStackedWidget,
QWidget,
QLabel,
QToolButton,
QMenu,
QWidgetAction,
QStyle,
QApplication,
QMessageBox,
QListWidgetItem)
from backend.barista.utils.logger import Log
from backend.parser.parser import Parser
from gui.main_window.docks.dock import DockElement
from gui.main_window.docks.plotter.qt5_plotter import Qt5Plotter
class DockElementPlotter(DockElement):
selected = None
registerKeySignal = pyqtSignal(str, str, str)
__settings = {}
def __init__(self, mainWindow, title):
DockElement.__init__(self, mainWindow, 'Plotter')
self.name = title
self.setWindowTitle(title)
self.setupUi()
def setupUi(self):
self.resize(500, 300)
self.settingsWidget = QWidget()
self.settingsWidget.setMinimumSize(900,150)
# Create the main widget as a container widget.
self.mainWidget = QWidget(self)
self.setWidget(self.mainWidget)
mainLayout = QVBoxLayout(self.mainWidget)
# Create 'Settings' dropdown button.
self.settingsButton = QToolButton(self)
self.settingsButton.setText("Settings")
# Remove the menu indicator by applying a global style.
self.settingsButton.setObjectName("noIndicator")
# And replace it with a down arrow icon instead (which looks better).
self.settingsButton.setArrowType(QtCore.Qt.DownArrow)
self.settingsButton.setToolButtonStyle(QtCore.Qt.ToolButtonTextBesideIcon)
# Set the popup that opens on button click.
self.settingsButton.setPopupMode(QToolButton.InstantPopup)
action = QWidgetAction(self.settingsButton)
action.setDefaultWidget(self.settingsWidget)
self.settingsButton.addAction(action)
# Create a horizontal layout for the settings button.
buttonLayout = QHBoxLayout()
buttonLayout.addWidget(self.settingsButton)
buttonLayout.addStretch()
mainLayout.addLayout(buttonLayout)
# Create the plotter widget.
self.plotter = Qt5Plotter()
self.plotter.setPlotterGUI(self)
self.plotter.setMinimumSize(300,200)
self.plotter.plotLinear()
mainLayout.addWidget(self.plotter)
# Create the settings dialog.
self.setupSettingsDialogUi()
# Set keys that are plotted by default
self.defaultKeys = {"TEST": ["loss"], "TRAIN": ["loss"]}
def setupSettingsDialogUi(self):
self.settingsDialogLayout = QGridLayout()
# DropDown for plotAgainstTime or Iterations
self.dbTimeIter = QComboBox()
self.dbTimeIter.addItem("Iterations")
self.dbTimeIter.addItem("Time")
self.dbTimeIter.setFixedWidth(130)
self.dbTimeIter.currentIndexChanged.connect(self.plotAgainstTimeIter)
self.settingsDialogLayout.addWidget(self.dbTimeIter, 1, 3)
# DropDown for plotting scale linear or logarithmic
self.dbLinLoga = QComboBox()
self.dbLinLoga.addItem("Linear")
self.dbLinLoga.addItem("Logarithmic")
self.dbLinLoga.setFixedWidth(130)
self.dbLinLoga.currentIndexChanged.connect(self.plotLinLoga)
self.settingsDialogLayout.addWidget(self.dbLinLoga, 3,3)
# Dynamic checkboxes
# Key lists from parser that define the table rows for train and test
self.test_dict_list = []
self.train_dict_list = []
# Lists for checkboxes for train and test
self.test_cb = []
self.train_cb = []
# Text labels for better display
testLabel = QLabel("Test keys:")
trainLabel = QLabel("Train keys:")
timeIterLabel = QLabel("Plot against:")
LinLogaLabel = QLabel("Plot scale:")
self.settingsDialogLayout.addWidget(testLabel,0,4)
self.settingsDialogLayout.addWidget(trainLabel,2,4)
self.settingsDialogLayout.addWidget(timeIterLabel,0,3)
self.settingsDialogLayout.addWidget(LinLogaLabel,2,3)
# create default Checkboxes
self.createCheckboxes(self.test_dict_list, self.train_dict_list)
# Button select Logfiles
self.btn1 = QPushButton(QtGui.QIcon(QApplication.style().standardIcon(QStyle.SP_DialogOpenButton)), "")
self.btn1.setFixedWidth(40)
self.btn1.clicked.connect(self.getFiles)
self.settingsDialogLayout.addWidget(self.btn1,3,0)
# Button remove item
self.btn2 = QPushButton(QtGui.QIcon(QApplication.style().standardIcon(QStyle.SP_TrashIcon)), "")
self.btn2.setFixedWidth(40)
self.btn2.clicked.connect(self.removeItem)
self.settingsDialogLayout.addWidget(self.btn2,3,1)
# Button CSV export
self.btn3 = QPushButton("Export as CSV")
self.btn3.setFixedWidth(110)
self.btn3.clicked.connect(self.exportCSV)
self.settingsDialogLayout.addWidget(self.btn3,3,2)
# Create Logfile list
self.leftlist = QListWidget()
self.leftlist.setFixedSize(200,100)
self.Stack = QStackedWidget(self)
self.settingsDialogLayout.addWidget(self.Stack,0,0,3,3)
self.settingsDialogLayout.addWidget(self.leftlist,0,0,3,3)
self.leftlist.currentRowChanged.connect(self.display)
self.leftlist.currentItemChanged.connect(self.selectedLogItem)
self.settingsWidget.setLayout(self.settingsDialogLayout)
# Signal handling
self.registerKeySignal.connect(self.registerKey)
def btnstatetest(self, checkbox):
"""
Plot the selected checkbox for Test-Data
"""
if self.selected:
self.plotter.showMetricTest(self.selected, checkbox.text(), checkbox.isChecked())
self.displayPlot()
self.__settings["checkBoxes"][self.selected]["TEST"][checkbox.text()] = checkbox.isChecked()
def btnstatetrain(self, checkbox):
"""
Plot the selected checkbox for Train-Data
"""
if self.selected:
self.plotter.showMetricTrain(self.selected, checkbox.text(), checkbox.isChecked())
self.displayPlot()
self.__settings["checkBoxes"][self.selected]["TRAIN"][checkbox.text()] = checkbox.isChecked()
def displayPlot(self):
"""
Update the plotter.
"""
if self.selected:
self.plotter.plot()
self.plotter.show()
self.createNecessaryDocks()
def createNecessaryDocks(self):
"""
Create the necessary docks if they don't exist.
"""
if self.__settings is None:
self.__settings = {}
if "checkBoxes" not in self.__settings:
self.__settings["checkBoxes"] = {}
if self.selected not in self.__settings["checkBoxes"] or \
isinstance(self.__settings["checkBoxes"][self.selected], list):
self.__settings["checkBoxes"][self.selected] = {"TEST": {}, "TRAIN": {}}
def existsKeySelection(self):
"""
Returns true if there is a selection by the user of keys to be plotted
"""
if self.__settings is None:
self.__settings = {}
if "checkBoxes" not in self.__settings:
self.__settings["checkBoxes"] = {}
if self.selected not in self.__settings["checkBoxes"]:
return False
return True
def createCheckboxes(self, test_list, train_list):
"""
Dynamically creates checkboxes, which are in the lists: test_list and train_list
"""
self.test_dict_list = test_list
self.train_dict_list = train_list
# As long as there are elements in TEST list: create Checkboxes
if self.test_dict_list is not None:
for i in range(len(self.test_dict_list)):
self.test_cb.append(QCheckBox(self.test_dict_list[i], self))
self.settingsDialogLayout.addWidget(self.test_cb[i], 1, 4+i)
self.test_cb[i].stateChanged.connect(lambda checked, i=i: self.btnstatetest(self.test_cb[i]))
if self.existsKeySelection() and self.test_cb[i].text() in \
self.__settings["checkBoxes"][self.selected]["TEST"]:
# Compatibility for older projects
try:
self.test_cb[i].setChecked(
self.__settings["checkBoxes"][self.selected]["TEST"].get(self.test_cb[i].text(), True))
except AttributeError:
self.test_cb[i].setChecked(True)
else:
self.test_cb[i].setChecked(self.test_cb[i].text() in self.defaultKeys["TEST"])
self.btnstatetest(self.test_cb[i])
# As long as there are elements in TRAIN list: create Checkboxes
if self.train_dict_list is not None:
for i in range(len(self.train_dict_list)):
self.train_cb.append(QCheckBox(self.train_dict_list[i], self))
self.settingsDialogLayout.addWidget(self.train_cb[i], 3, 4+i)
self.train_cb[i].stateChanged.connect(lambda checked, i=i: self.btnstatetrain(self.train_cb[i]))
if self.existsKeySelection() and self.train_cb[i].text() in \
self.__settings["checkBoxes"][self.selected]["TRAIN"]:
# Compatibility for older projects
try:
self.train_cb[i].setChecked(
self.__settings["checkBoxes"][self.selected]["TRAIN"].get(self.train_cb[i].text(), True))
except AttributeError:
self.train_cb[i].setChecked(True)
else:
self.train_cb[i].setChecked(self.train_cb[i].text() in self.defaultKeys["TRAIN"])
self.btnstatetrain(self.train_cb[i])
self.settingsWidget.setLayout(self.settingsDialogLayout)
def display(self, i):
self.Stack.setCurrentIndex(i)
def loadFiles(self, filenames):
"""
Loads every log file in the list of filenames and adds them to the list of logs.
"""
for i in range(len(filenames)):
f = filenames[i]
if not os.path.exists(f):
Log.error("External log not found: " + f, Log.getCallerId("plotter"))
else:
parser = Parser(open(f, 'r'), OrderedDict(), Log.getCallerId(str(f)))
head, tail = os.path.split(str(f))
logId = "external_" + tail
logName = "[ext] " + tail
self.putLog(logId, parser, logName=logName)
parser.parseLog()
# This is for saving the loaded logs in the project file
# (create the necessary docks if they don't exist)
if self.__settings is None:
self.__settings = {"logFiles": {logId: f}}
elif "logFiles" not in self.__settings:
self.__settings["logFiles"] = {logId: f}
else:
self.__settings["logFiles"][logId] = f
def getFiles(self):
"""
Dialog for adding different data files to list for plotting
"""
dlg = QFileDialog()
dlg.setFileMode(QFileDialog.ExistingFiles)
if dlg.exec_():
self.loadFiles(dlg.selectedFiles())
self.selectLast()
def plotAgainstTimeIter(self):
if self.dbTimeIter.currentText() == "Time":
self.plotter.plotAgainstTime()
self.__settings["againstTime"] = "Time"
self.plotter.plot()
else:
if self.dbTimeIter.currentText() == "Iterations":
self.plotter.plotAgainstIterations()
self.__settings["againstTime"] = "Iterations"
self.plotter.plot()
def plotLinLoga(self):
if self.dbLinLoga.currentText() == "Linear":
self.plotter.plotLinear()
self.__settings["logarithmic"] = "Linear"
self.plotter.plot()
else:
if self.dbLinLoga.currentText() == "Logarithmic":
self.plotter.plotLogarithmic()
self.__settings["logarithmic"] = "Logarithmic"
self.plotter.plot()
def putLog(self, logId, parser, plotOnUpdate=False, logName=None):
"""
Add a log (i.e. parser) to the list of shown logs. If a log with the
log name already exists primes are added to the name until it is unique.
"""
for i in range(self.leftlist.count()):
if logId == self.leftlist.item(i).data(Qt.UserRole):
return
if not logName:
logName = logId
self.test_dict_list = []
self.train_dict_list = []
while self.leftlist.findItems(logName, Qt.MatchExactly):
logName += "'"
self.plotter.putLog(logId, logName, parser, plotOnUpdate)
logItem = QListWidgetItem(logName)
v = QVariant(logId)
logItem.setData(Qt.UserRole, v)
self.leftlist.insertItem(self.leftlist.count(), logItem)
self.selectLast()
def removeLog(self, logId, logName):
try:
self.__settings["checkBoxes"].pop(logId)
except KeyError:
pass
self.plotter.removeLog(logId)
item = self.leftlist.findItems(logName, Qt.MatchExactly)
if item:
row = self.leftlist.row(item[0])
self.leftlist.takeItem(row)
self.test_dict_list = []
self.train_dict_list = []
self.selectLast()
self.plotter.plot()
self.plotter.show()
def registerKey(self, logId, phase, key):
"""
Registers the key given by the signal. This is only of interest if the key belongs
to the selected log file. A new checkbox is then created for that key.
"""
if self.leftlist.currentItem():
if str(self.leftlist.currentItem().data(Qt.UserRole)) == logId:
if phase == "TEST":
self.test_dict_list.append(key)
if phase == "TRAIN":
self.train_dict_list.append(key)
self.updateCheckboxes(self.test_dict_list, self.train_dict_list)
def removeItem(self):
"""
Removes the selected item from the list and also from plot.
"""
item = self.leftlist.takeItem(self.leftlist.currentRow())
if item is not None:
self.removeLog(item.data(Qt.UserRole), item.text())
self.displayPlot()
self.selectLast()
if (self.__settings is not None
and "logFiles" in self.__settings
and str(item.data(Qt.UserRole)) in self.__settings["logFiles"]):
del self.__settings["logFiles"][str(item.data(Qt.UserRole))]
def selectLast(self):
if len(self.leftlist) > 0:
self.leftlist.setCurrentItem(self.leftlist.item(len(self.leftlist) - 1))
self.selectedLogItem(self.leftlist.item(len(self.leftlist) - 1))
else:
self.test_dict_list = []
self.train_dict_list = []
self.updateCheckboxes(self.test_dict_list,self.train_dict_list)
def selectedLogItem(self, item, update=True):
"""
Memorize which logfile is selected, block signals, and asks plotter if the checkbox is
selected or not.
"""
if item:
if update:
self.updateCheckboxes(
list(self.plotter.getParser(str(item.data(Qt.UserRole))).getKeys('TEST')),
list(self.plotter.getParser(str(item.data(Qt.UserRole))).getKeys('TRAIN')))
self.selected = str(str(item.data(Qt.UserRole)))
self.tickBoxes()
self.__settings["selectedIndex"] = self.leftlist.row(item)
def updateCheckboxes(self, test_list, train_list):
"""
Clear all checkboxes and clear the layer and create new checkboxes and
add them to the layer again
"""
for i in range(len(self.test_cb)):
self.settingsDialogLayout.removeWidget(self.test_cb[i])
self.test_cb[i].deleteLater()
self.test_cb[i] = None
for i in range(len(self.train_cb)):
self.settingsDialogLayout.removeWidget(self.train_cb[i])
self.train_cb[i].deleteLater()
self.train_cb[i] = None
self.test_cb = []
self.train_cb = []
# create new Checkboxes with new Keys
self.createCheckboxes(test_list, train_list)
self.tickBoxes()
def tickBoxes(self):
"""
Whether a plot is shown is stored in the plotter object. This method
ticks the checkboxes of the keys whose plot is shown in the plotter.
"""
for i in range(len(self.test_cb)):
self.test_cb[i].blockSignals(True)
self.test_cb[i].setChecked(self.plotter.isMetricTestShown(self.selected, self.test_cb[i].text()))
self.test_cb[i].blockSignals(False)
for i in range(len(self.train_cb)):
self.train_cb[i].blockSignals(True)
self.train_cb[i].setChecked(self.plotter.isMetricTrainShown(self.selected, self.train_cb[i].text()))
self.train_cb[i].blockSignals(False)
def setProject(self, project):
"""
This sets the project object. The plotter settings have the following structure:
"plotter": {
"checkBoxes": {
"logName1": {
"TEST": {
"accuracy": true,
"loss": false
},
"TRAIN": {
"LearningRate": true,
"loss": false
}
},
"logName2": {
...
},
...
},
"selectedIndex": 0,
"againstTime": "Iterations",
"logarithmic": "Logarithmic"
}
"""
if "plotter" not in project.getSettings():
project.getSettings()["plotter"] = self.__settings
else:
self.__settings = project.getSettings()["plotter"]
selectedIndex = -1
if "selectedIndex" in self.__settings:
selectedIndex = self.__settings["selectedIndex"]
if "logFiles" in self.__settings:
self.loadFiles(list(self.__settings["logFiles"].values()))
if "checkBoxes" in self.__settings:
for name in self.__settings["checkBoxes"]:
for key in self.__settings["checkBoxes"][name]["TEST"]:
# Compatibility for older projects
try:
self.plotter.showMetricTest(name, key, self.__settings["checkBoxes"][name]["TEST"].get(key))
except AttributeError:
self.plotter.showMetricTest(name, key, True)
for key in self.__settings["checkBoxes"][name]["TRAIN"]:
# Compatibility for older projects
try:
self.plotter.showMetricTrain(name, key, self.__settings["checkBoxes"][name]["TRAIN"].get(key))
except AttributeError:
self.plotter.showMetricTrain(name, key, True)
if "againstTime" in self.__settings:
index = self.dbTimeIter.findText(self.__settings["againstTime"], Qt.MatchFixedString)
if index >= 0:
self.dbTimeIter.setCurrentIndex(index)
if "logarithmic" in self.__settings:
index = self.dbLinLoga.findText(self.__settings["logarithmic"], Qt.MatchFixedString)
if index >= 0:
self.dbLinLoga.setCurrentIndex(index)
#for sid in project.getSessions():
# session = project.getSession(sid)
# self.putLog(str(session.getLogId()), session.getParser(), True, str(session.getLogFileName(True)))
self.plotter.plot()
if selectedIndex >= 0:
self.leftlist.setCurrentItem(self.leftlist.item(selectedIndex))
self.selectedLogItem(self.leftlist.item(selectedIndex), False)
else:
self.selectLast()
# Delete plot if session is deleted
project.deleteSession.connect(lambda sid: self.removeLog(str(project.getSession(sid).getLogId()),
str(project.getSession(sid).getLogFileName(True))))
project.deleteSession.connect(lambda sid: project.deletePlotterSettings(str(project.getSession(sid).getLogId())))
project.resetSession.connect(lambda sid: self.removeLog(str(project.getSession(sid).getLogId()),
str(project.getSession(sid).getLogFileName(True))))
project.resetSession.connect(lambda sid: project.deletePlotterSettings(str(project.getSession(sid).getLogId())))
def exportCSV(self):
"""
Opens a file dialog and the plotter saves the shown plots to the selected file.
The file dialog only appears if there is at least one plot.
"""
if self.plotter.numGraphs() == 0:
QMessageBox.question(self, 'PyQt5 message', "Nothing selected!", QMessageBox.Ok, QMessageBox.Ok)
else:
path = str(QFileDialog.getSaveFileName(filter="CSV table (*.csv) ;; Text file (*.txt)")[0])
if path:
if not (path.endswith(".csv") or path.endswith(".txt")):
path = path + ".csv"
self.plotter.exportCSVToFile(path)
Barista/gui/main_window/docks/properties/__init__.py 0000664 ڊ0011117 00000000000 13265130053 023061 0 ustar klemms o0cvpr Barista/gui/main_window/docks/properties/dock_layer_properties.py 0000664 ڊ0011117 00000004723 13265130053 025732 0 ustar klemms o0cvpr from PyQt5 import QtWidgets
from gui.main_window.docks.dock import DockElement
from gui.network_manager.network_manager import NetworkManager
from gui.main_window.docks.properties.properties_data_model import PropertiesDataModel
from gui.main_window.docks.properties.properties_widget import PropertiesWidget
class LayerProperties(DockElement):
def __init__(self, mainWindow, title):
DockElement.__init__(self, mainWindow, "Layer Properties")
self.tab = QtWidgets.QTabWidget()
self.tab.setDocumentMode(True)
self.setWidget(self.tab)
self.ids = []
self.datadict = {}
self._mainWindow = mainWindow
self.disableEditing(mainWindow.getDisabled())
def setTab(self, id):
""" Set the layer whose properties should be shown by its id """
self.tab.clear()
self.addTab(id)
def addTab(self, id):
""" Add a new property editor for the layer with the given id """
network = self._networkmanager.network
falloutlen = len(network.uri())
layer = network["layers"][id]
data = layer["parameters"]
widget = PropertiesWidget(self)
widget.setModel(PropertiesDataModel(data))
self.tab.addTab(widget, data["name"])
self.ids.append(id)
self.datadict[id] = data
self.disableEditing(self._mainWindow.getDisabled())
def setNetworkManager(self, networkManager):
self._networkmanager = networkManager # type: NetworkManager
def setTabs(self, ids):
""" Set the layers whose properties should be shown by its id.
ids is a list of string.
"""
self.clearProperties()
for id in ids:
self.addTab(id)
def clearProperties(self):
""" Let the dock show no properties for any layer """
self.ids = []
self.datadict = {}
self.tab.clear()
def updateDock(self):
""" Rebuild all property editors in this dock """
ids = self.ids
self.clearProperties()
self.setTabs(ids)
def removeTab(self, id):
""" Remove the property editor for the layer with the given id """
idx = self.ids.index(id)
self.tab.removeTab(idx)
self.ids.remove(id)
del self.datadict[id]
def disableEditing(self, disable):
""" Disable Editing of each widget in this dock """
for i in range(0, self.tab.count()):
tab = self.tab.widget(i)
tab.disableEditing(disable)
Barista/gui/main_window/docks/properties/dock_solver_properties.py 0000664 ڊ0011117 00000003062 13265130053 026123 0 ustar klemms o0cvpr from PyQt5 import QtCore, QtWidgets, QtGui
from PyQt5.QtCore import pyqtSignal
from gui.main_window.docks.properties.properties_data_model import PropertiesDataModel
from gui.main_window.docks.properties.properties_widget import PropertiesWidget
from gui.main_window.docks.dock import DockElement
class SolverProperties(DockElement):
# This signal will be emitted when any of the properties of the solver change.
solverChanged = pyqtSignal(list, object)
def __init__(self, mainWindow, title):
DockElement.__init__(self, mainWindow, "Solver Properties")
self._data = None
self.setToolTip('This shows the Properties the loaded Solver.')
self.setupUi()
self._mainWindow = mainWindow
self.disableEditing(mainWindow.getDisabled())
def setupUi(self):
self.tree = PropertiesWidget(self)
self.setWidget(self.tree)
def setSolver(self, reactiveSolver):
""" Set the solver whose properties should be shown.
reactiveSolver has to be an instance of PropertyData
"""
if self._data:
self._data.someChildPropertyChanged.disconnect(self.solverChanged)
self._data = reactiveSolver
self.tree.setModel(PropertiesDataModel(self._data, self.tree))
self._data.someChildPropertyChanged.connect(self.solverChanged)
self.disableEditing(self._mainWindow.getDisabled())
def data(self):
return self._data
def disableEditing(self, disable):
""" Disable Editing of this dock """
self.tree.disableEditing(disable)
Barista/gui/main_window/docks/properties/properties_data_model.py 0000664 ڊ0011117 00000024433 13265130053 025707 0 ustar klemms o0cvpr from PyQt5 import QtCore, QtGui, QtWidgets
from PyQt5.Qt import QApplication
from PyQt5.QtCore import QModelIndex, QVariant
from gui.main_window.docks.properties.property_widgets.data import *
from gui.main_window.docks.properties.property_widgets.types import *
def getChild(data, row):
if isinstance(data, PropertyDataGroupObject):
data = data.value()
if isinstance(data, PropertyData):
key = sortPropertyKeys(data.getBoringDict().keys())[row]
c = data.getChildDataObject(key)
elif isinstance(data, PropertyDataDictObject):
key = sortPropertyKeys(data.value().keys())[row]
c = data.value()[key]
else:
c = data.value()[row]
return c
def getChildCount(data):
if isinstance(data, PropertyDataGroupObject):
data = data.value()
if isinstance(data, PropertyData):
return len(data.givenProperties())
elif isinstance(data, PropertyDataListObject):
return len(data.value())
elif isinstance(data, PropertyDataDictObject):
return len(data.value())
return 0
def getColumnCount(data):
return 2
def getParent(data, root):
# IMPORTANT: PropertyDataGroupObject and contained PropertyData have the same
# uri.
if root == data:
return None
# Find uri from root to parent.
uri = data.uri()[len(root.uri()):-1]
for key in uri:
try:
if isinstance(root, PropertyData):
root = root.getChildDataObject(key)
elif isinstance(root, PropertyDataGroupObject):
root = root.value().getChildDataObject(key)
else:
root = root.value()[key]
except:
print("[ERROR] Key '%s' unexpectedly not found in property '%s'." % (str(key), str(root.value())))
return None
return root
def getRow(data, root):
parent = getParent(data, root)
key = data.uri()[-1]
if isinstance(parent, PropertyData):
return sortPropertyKeys(parent.getBoringDict().keys()).index(key)
elif isinstance(parent, PropertyDataDictObject):
return sortPropertyKeys(parent.values().keys()).index(key)
elif isinstance(parent, PropertyDataListObject):
return key
return 0
def getName(data):
return str(data.uri()[-1])
def getValue(data):
if type(data) != PropertyDataObject:
return ''
return data.value()
def isRequired(data):
return not isinstance(data, PropertyDataObject) or data.info().isRequired()
def isEditable(data):
# if type(data) == PropertyDataObject:
# print(data.info())
return type(data) == PropertyDataObject and data.info().isEditable()
def hasProperty(data, key):
try:
return key in data
except:
return False
def sortPropertyKeys(keys):
# Sort the keys normally.
result = sorted(keys)
# Then bring the 'name' and 'type' properties to the front of the list if
# they are available.
if 'top' in result:
result.insert(-1, result.pop(result.index('top')))
if 'bottom' in result:
result.insert(-1, result.pop(result.index('bottom')))
if 'type' in result:
result.insert(0, result.pop(result.index('type')))
if 'name' in result:
result.insert(0, result.pop(result.index('name')))
return result
class PropertiesDataModel(QtCore.QAbstractItemModel):
def __init__(self, data, parent=None):
"""
data : PropertyData
"""
super(PropertiesDataModel, self).__init__(parent)
self.root = data
def data(self, index, role):
""" Only return display data. """
if not index.isValid():
return QVariant()
node = index.internalPointer()
dataList = node.uri()
if role == QtCore.Qt.DecorationRole and index.column() == 0:
if type(node) == PropertyDataObject:
if 'top' in dataList or 'bottom' in dataList:
return QtGui.QIcon('resources/blob.svg')
else:
return QtGui.QIcon('resources/property.svg')
# return QtGui.QIcon('resources/property.svg')
else:
if 'top' in dataList or 'bottom' in dataList:
return QtGui.QIcon('resources/blob.svg')
else:
return QtGui.QIcon('resources/group.svg')
# return QApplication.instance().style().standardIcon(getattr(QtWidgets.QStyle, 'SP_DirIcon'))
if role == QtCore.Qt.ForegroundRole and isinstance(node, PropertyDataObject) and not isEditable(node) and index.column() > 0:
return QtGui.QBrush(QtGui.QColor(125, 125, 125))
if role == QtCore.Qt.ToolTipRole and isinstance(node, PropertyDataObject):
if node.info().description() == '':
return 'No info available.'
return '' + node.info().description() + ''
if role == QtCore.Qt.DisplayRole:
return getName(node) if index.column() == 0 else getValue(node)
return QVariant()
def rowCount(self, parent):
# If parent is invalid, we show the root node only, so we have 1 row.
if not parent.isValid():
return 1
parentNode = parent.internalPointer()
return getChildCount(parentNode)
def columnCount(self, index):
if not index.isValid():
return 2
# parentNode = self.root
else:
parentNode = index.internalPointer()
return getColumnCount(parentNode)
def flags(self, index):
if index.isValid() and index.column() > 0 and isEditable(index.internalPointer()):
return QtCore.Qt.ItemIsEditable | super(PropertiesDataModel, self).flags(index)
return super(PropertiesDataModel, self).flags(index)
def headerData(self, section, orientation, role):
if orientation == QtCore.Qt.Horizontal and role == QtCore.Qt.DisplayRole:
if section == 0:
return "Property"
if section == 1:
return "Value"
return QVariant()
def index(self, row, column, parent):
if not self.hasIndex(row, column, parent):
return QModelIndex()
if not parent.isValid():
return self.createIndex(row, column, self.root)
# parentNode = self.root
else:
parentNode = parent.internalPointer()
childNode = getChild(parentNode, row)
if childNode is not None:
return self.createIndex(row, column, childNode)
return QModelIndex()
def parent(self, index):
if not index.isValid():
return QModelIndex()
childNode = index.internalPointer()
parentNode = getParent(childNode, self.root)
if parentNode is None:
return QModelIndex()
row = getRow(parentNode, self.root)
if row is None:
return QModelIndex()
return self.createIndex(row, 0, parentNode)
# Below is everything that's required for writing to the model.
def setData(self, index, value, role):
if not index.isValid():
return False
node = index.internalPointer()
if type(node) != PropertyDataObject or role != QtCore.Qt.EditRole:
return False
try:
if index.internalPointer().info().typeString() == IntType:
value = int(value)
elif index.internalPointer().info().typeString() == FloatType:
value = float(value)
elif index.internalPointer().info().typeString() == BoolType:
value = bool(value)
# TODO: EnumType
index.internalPointer().setValue(value)
self.dataChanged.emit(index, index)
except:
print('[ERROR] failed to set value of %s, because it had an invalid type.' % (getName(node)))
return False
return True
def setHeaderData(self, section, orientation, value, role):
return NotImplementedError()
def addToGroup(self, parent, key, value=None):
# Make sure that we add this as a child of the first column, to prevent
# GUI update issues.
parent = self.index(parent.row(), 0, parent.parent())
parentNode = parent.internalPointer()
if not isinstance(parentNode, PropertyDataGroupObject) and not parentNode == self.root:
return False
node = parentNode if parentNode == self.root else parentNode.value()
keys = node.getBoringDict().keys()
if key in keys:
return False
keys.append(key)
row = sortPropertyKeys(keys).index(key)
self.beginInsertRows(parent, row, row)
if value is None:
node.givePropertyDefault(key)
else:
node.giveProperty(key, value)
self.endInsertRows()
return True
def addToList(self, parent, value):
# Make sure that we add this as a child of the first column, to prevent
# GUI update issues.
parent = self.index(parent.row(), 0, parent.parent())
parentNode = parent.internalPointer()
if not isinstance(parentNode, PropertyDataListObject):
return False
row = len(parentNode.value())
self.beginInsertRows(parent, row, row)
parentNode.pushBack(value)
self.endInsertRows()
return True
def removeNode(self, index):
node = index.internalPointer()
parent = self.parent(index)
parentNode = getParent(node, self.root)
# We are only allowed to remove nodes when the parent is root, or and
# instance of PropertyDataGroupObject or PropertyDataListObject.
if parentNode == self.root:
self.beginRemoveRows(parent, index.row(), index.row())
parentNode.ungiveProperty(node._idx)
self.endRemoveRows()
elif isinstance(parentNode, PropertyDataGroupObject):
self.beginRemoveRows(parent, index.row(), index.row())
parentNode.value().ungiveProperty(node._idx)
self.endRemoveRows()
elif isinstance(parentNode, PropertyDataListObject):
self.beginRemoveRows(parent, index.row(), index.row())
parentNode.removeObject(node._idx)
self.endRemoveRows()
# Otherwise we print an error message.
else:
print("[ERROR] Cannot remove property from node of type %s" % str(type(parentNode)))
return False
Barista/gui/main_window/docks/properties/properties_delegate.py 0000664 ڊ0011117 00000007427 13265130053 025374 0 ustar klemms o0cvpr from PyQt5 import QtCore, QtGui, QtWidgets
from PyQt5.QtCore import QModelIndex, QVariant
from gui.main_window.docks.properties.property_widgets.data import *
from gui.main_window.docks.properties.property_widgets.types import *
class PropertiesDelegate(QtWidgets.QStyledItemDelegate):
def __init__(self, parent=None):
super(PropertiesDelegate, self).__init__(parent)
def createEditor(self, parent, options, index):
if index.column() == 0:
return super(PropertiesDelegate, self).createEditor(parent, options, index)
if index.internalPointer().info().typeString() == EnumType:
return self.createEnumEditor(parent, options, index)
elif index.internalPointer().info().typeString() == FloatType:
return self.createFloatEditor(parent, options, index)
elif index.internalPointer().info().typeString() == IntType:
return self.createIntEditor(parent, options, index)
elif index.internalPointer().info().typeString() == BoolType:
return self.createBoolEditor(parent, options, index)
return super(PropertiesDelegate, self).createEditor(parent, options, index)
def createEnumEditor(self, parent, options, index):
data = index.internalPointer()
comboBox = QtWidgets.QComboBox(parent)
comboBox.addItems(data.info().enumOptions())
comboBox.setCurrentText(data.value())
return comboBox
def createBoolEditor(self, parent, options, index):
data = index.internalPointer()
comboBox = QtWidgets.QComboBox(parent)
comboBox.addItem("true")
comboBox.addItem("false")
comboBox.setCurrentText(str(data.value()).lower())
return comboBox
def createFloatEditor(self, parent, options, index):
data = index.internalPointer()
spinBox = QtWidgets.QDoubleSpinBox(parent)
spinBox.setRange(-pow(2, 31), pow(2, 31-1))
spinBox.setFocusPolicy(QtCore.Qt.StrongFocus )
spinBox.setDecimals(15)
spinBox.setSingleStep(0.05)
spinBox.setValue(data.value())
return spinBox
def createIntEditor(self, parent, options, index):
data = index.internalPointer()
spinBox = QtWidgets.QSpinBox(parent)
spinBox.setRange(-pow(2, 31), pow(2, 31-1))
spinBox.setFocusPolicy(QtCore.Qt.StrongFocus )
spinBox.setValue(data.value())
return spinBox
def setEditorData(self, editor, index):
data = index.internalPointer()
if isinstance(editor, QtWidgets.QComboBox):
if index.internalPointer().info().typeString() == BoolType:
editor.setCurrentText(str(data.value()).lower())
else:
editor.setCurrentText(data.value())
elif isinstance(editor, QtWidgets.QSpinBox) or isinstance(editor, QtWidgets.QDoubleSpinBox):
editor.setValue(data.value())
elif isinstance(editor, QtWidgets.QLineEdit):
editor.setText(data.value())
else:
super(PropertiesDelegate, self).setEditorData(editor, index)
def setModelData(self, editor, model, index):
if isinstance(editor, QtWidgets.QComboBox):
if index.internalPointer().info().typeString() == BoolType:
model.setData(index, True if editor.currentText() == "true" else False, QtCore.Qt.EditRole)
else:
model.setData(index, editor.currentText(), QtCore.Qt.EditRole)
elif isinstance(editor, QtWidgets.QSpinBox) or isinstance(editor, QtWidgets.QDoubleSpinBox):
model.setData(index, editor.value(), QtCore.Qt.EditRole)
elif isinstance(editor, QtWidgets.QLineEdit):
model.setData(index, editor.text(), QtCore.Qt.EditRole)
else:
super(PropertiesDelegate, self).setModelData(editor, model, index)
Barista/gui/main_window/docks/properties/properties_suggestion_model.py 0000664 ڊ0011117 00000004144 13265130053 027162 0 ustar klemms o0cvpr from PyQt5 import QtCore, QtGui
from PyQt5.QtCore import QModelIndex, QVariant
from gui.main_window.docks.properties.property_widgets.data import *
class PropertiesSuggestionModel(QtCore.QAbstractListModel):
def __init__(self, data=None, parent=None):
super(PropertiesSuggestionModel, self).__init__(parent)
self.data = data
def newProperties(self):
""" Return a list of properties which are not
set in the GroupData
"""
if self.data is None:
return []
# Find suggestions as list of available properties that have not been given.
availableProperties = self.data.allAvailableProperties()
givenProperties = [prop.info().name() for prop in self.data.givenProperties()]
suggestions = [prop for name, prop in availableProperties.items() if name not in givenProperties]
# Sort properties alphabetically.
suggestions.sort(key=lambda x: x.name())
# Move deprecated properties to the end of the list.
suggestionsFinal = []
suggestionsDeprecated = []
for prop in suggestions:
if prop.deprecated():
suggestionsDeprecated.append(prop)
else:
suggestionsFinal.append(prop)
suggestionsFinal.extend(suggestionsDeprecated)
return suggestionsFinal
def rowCount(self, index):
return len(self.newProperties())
def data(self, index, role):
if not index.isValid():
return QVariant()
row = index.row()
if row >= len(self.newProperties()):
return QVariant()
prop = self.newProperties()[row]
if role == QtCore.Qt.ToolTipRole:
if prop.description() == '':
return 'No info available.'
return '' + prop.description() + ''
if role == QtCore.Qt.DisplayRole or role == QtCore.Qt.EditRole:
return prop.name() + ' [' + prop.typeString() + ']'
if role == QtCore.Qt.UserRole:
return prop
return QVariant()
def update(self):
self.modelReset.emit()
Barista/gui/main_window/docks/properties/properties_widget.py 0000664 ڊ0011117 00000013540 13265130053 025076 0 ustar klemms o0cvpr from PyQt5 import QtCore, QtGui, QtWidgets
from PyQt5.QtCore import QModelIndex, QVariant
from PyQt5.QtWidgets import QAbstractItemView
from gui.main_window.docks.properties.properties_delegate import PropertiesDelegate
from gui.main_window.docks.properties.properties_data_model import PropertiesDataModel
from gui.main_window.docks.properties.properties_suggestion_model import PropertiesSuggestionModel
from gui.main_window.docks.properties.property_widgets.data import *
class PropertiesWidget(QtWidgets.QWidget):
def __init__(self, parent=None):
super(PropertiesWidget, self).__init__(parent)
self.disabled = False
self.setupUi()
def setupUi(self):
mainLayout = QtWidgets.QVBoxLayout(self)
# Create the treeview to display the properties.
self.model = None
self.tree = QtWidgets.QTreeView(self)
self.tree.setObjectName("properties")
self.tree.setItemDelegate(PropertiesDelegate(self.tree))
self.tree.setEditTriggers(QtWidgets.QAbstractItemView.AllEditTriggers)
self.tree.header().setStretchLastSection(False)
self.tree.header().setSectionResizeMode(QtWidgets.QHeaderView.Stretch)
mainLayout.addWidget(self.tree)
# Add a label that shows node descriptions.
self.descriptionLabel = QtWidgets.QLabel("", self)
mainLayout.addWidget(self.descriptionLabel)
self.descriptionLabel.setObjectName('infoText')
self.descriptionLabel.setWordWrap(True)
# Create buttons to add and remove properties.
buttonLayout = QtWidgets.QHBoxLayout()
buttonLayout.addStretch()
mainLayout.addLayout(buttonLayout)
# Add ComboBox to select properties.
self.propertyBox = QtWidgets.QComboBox(self)
self.propertyBox.setModel(PropertiesSuggestionModel())
buttonLayout.addWidget(self.propertyBox, 1)
# Add Buttons to add and remove properties.
self.addButton = QtWidgets.QPushButton("Add", self)
self.addButton.clicked.connect(self.addClickedHandler)
buttonLayout.addWidget(self.addButton, 0)
self.removeButton = QtWidgets.QPushButton("Remove", self)
buttonLayout.addWidget(self.removeButton, 0)
self.removeButton.clicked.connect(self.removeClickedHandler)
self.rowChangedHandler(QModelIndex(), QModelIndex())
def setModel(self, model):
self.model = model
self.tree.setModel(model)
self.tree.selectionModel().currentRowChanged.connect(self.rowChangedHandler)
self.tree.expandAll()
self.tree.setCurrentIndex(self.model.index(0, 0, QModelIndex()))
def rowChangedHandler(self, current, previous):
node = current.internalPointer()
isRoot = node is not None and node == self.model.root
self.propertyBox.setEnabled((isinstance(node, PropertyDataGroupObject) or isRoot) and not self.disabled)
self.propertyBox.setVisible(isinstance(node, PropertyDataGroupObject) or isRoot)
# disable add and remove buttons in property widget
if ((isinstance(node, PropertyDataListObject) and ("top" in node.uri() or "bottom" in node.uri()))) \
or (isinstance(node, PropertyDataObject) and ("top" in node.uri() or "bottom" in node.uri())):
self.addButton.setDisabled(True)
self.removeButton.setDisabled(True)
else:
self.addButton.setEnabled((not current.isValid() or isinstance(node, PropertyDataGroupObject) or isRoot or isinstance(node, PropertyDataListObject)) and not self.disabled)
self.removeButton.setEnabled(current.isValid() and not isRoot and not node.info().isRequired() and not self.disabled)
if isinstance(node, PropertyDataGroupObject):
self.propertyBox.setModel(PropertiesSuggestionModel(node.value()))
elif isRoot:
self.propertyBox.setModel(PropertiesSuggestionModel(node))
else:
self.propertyBox.setModel(PropertiesSuggestionModel())
# Show description.
if node is not None:
if isinstance(node, PropertyDataObject):
self.descriptionLabel.setText(node.info().description())
else:
self.descriptionLabel.setText('')
def addClickedHandler(self):
index = self.tree.selectionModel().currentIndex()
if not index.isValid():
return
node = index.internalPointer()
isRoot = node is not None and node == self.model.root
# Add to group or root.
if isinstance(node, PropertyDataGroupObject) or isRoot:
prop = self.propertyBox.currentData(QtCore.Qt.UserRole)
try:
self.model.addToGroup(index, prop.name(), prop.prototype())
except Exception as e:
QtWidgets.QMessageBox.critical(self,
self.tr("Failed to add property."),
self.tr(type(e).__name__ + ": " + str(e)))
return
self.propertyBox.model().update()
# Add to list.
elif isinstance(node, PropertyDataListObject):
value = node.prototypeValue()
self.model.addToList(index, value)
def removeClickedHandler(self):
index = self.tree.selectionModel().currentIndex()
if index.isValid() and not index.internalPointer().info().isRequired():
self.model.removeNode(index)
self.propertyBox.model().update()
def disableEditing(self, disable):
""" Disable Editing of the tree """
self.disabled = disable
if disable:
self.tree.setEditTriggers(QAbstractItemView.NoEditTriggers)
self.propertyBox.setDisabled(disable)
self.addButton.setDisabled(disable)
self.removeButton.setDisabled(disable)
else:
self.tree.setEditTriggers(QtWidgets.QAbstractItemView.AllEditTriggers)
Barista/gui/main_window/docks/session/__init__.py 0000664 ڊ0011117 00000000000 13265130053 022350 0 ustar klemms o0cvpr Barista/gui/main_window/docks/session/dock_session_list.py 0000664 ڊ0011117 00000015201 13265130053 024340 0 ustar klemms o0cvpr import os
from PyQt5.QtCore import QPoint, Qt, pyqtSignal
from PyQt5.QtGui import QCursor
from PyQt5.QtGui import QStandardItem
from PyQt5.QtWidgets import (
QWidget,
QPushButton,
QMessageBox,
QComboBox,
QListWidget,
QToolButton,
QMenu,
QAction,
QVBoxLayout,
QHBoxLayout,
QListWidgetItem,
QAbstractItemView,
QFileDialog
)
import backend.caffe.loader as loader
from gui.main_window.docks.dock import DockElement
from gui.main_window.docks.session.session_item import SessionWidget
from gui.gui_util import askFromList
from backend.barista import caffe_versions
from backend.barista import project
class DockElementSessions(DockElement):
def __init__(self, mainWindow, title):
DockElement.__init__(self, mainWindow, title)
self.mainWindow = mainWindow
self.default_actions = mainWindow.actions
self.name = title
self.resize(300, 500)
self.__setupGui()
self.controller = None
self.displayed_sessions = None
def showDialog(self, text, info, title, buttons, location=None):
msg = QMessageBox()
msg.setIcon(QMessageBox.Information)
msg.setText(text)
msg.setInformativeText(info)
msg.setWindowTitle(title)
msg.setStandardButtons(buttons)
if location is None:
location = QCursor.pos()
msg.move(location - QPoint(msg.sizeHint().width() / 2,
msg.sizeHint().height() / 2))
return msg.exec_()
def __setupGui(self):
""" Create the widgets for this view.
"""
self.widget = QWidget()
vbox = QVBoxLayout()
hbox = QHBoxLayout()
#
#
# Create buttons
# create a new Session
self.new = QPushButton()
self.new.setToolTip('Create Session')
self.new.setText('New')
self.new.setEnabled(True)
self.new.clicked.connect(self._onCreate)
hbox.addWidget(self.new)
self.status_combo_box = QComboBox()
self.status_combo_box.clear()
self.status_combo_box.move(120, 30)
model = self.status_combo_box.model()
states = [
'ALL',
'RUNNING',
'PAUSED',
'WAITING',
'FAILED',
'FINISHED',
'INVALID'
]
for state in states:
item = QStandardItem(state)
model.appendRow(item)
hbox.addWidget(self.status_combo_box)
vbox.addLayout(hbox)
self.session_list = QListWidget()
self.session_list.setSelectionMode(QAbstractItemView.SingleSelection)
vbox.addWidget(self.session_list)
self.widget.setLayout(vbox)
self.setWidget(self.widget)
def showSessions(self, sessions):
""" Show the session widgets in the list.
"""
self.session_list.clear()
for sid in reversed(sorted(sessions.keys())):
session = sessions[sid]
item = QListWidgetItem(self.session_list)
item.setData(Qt.UserRole, sid)
widget = SessionWidget(session, self.controller, self.session_list)
item.setSizeHint(widget.sizeHint())
self.session_list.addItem(item)
self.session_list.setItemWidget(item, widget)
self.displayed_sessions = sessions
def sessionStateChanged(self, session, state):
""" Notify the widget of the given session about changed state.
"""
for item in self.session_list.findItems('', Qt.MatchRegExp):
widget = self.session_list.itemWidget(item)
if widget:
if widget.getSession() is session:
widget.sessionStateChanged(state)
def sessionIterationChanged(self, session):
""" Notify the widget of the given session about iteration-change.
"""
for item in self.session_list.findItems('', Qt.MatchRegExp):
widget = self.session_list.itemWidget(item)
if widget:
if widget.getSession() is session:
widget.sessionIterationChanged()
def sessionSnapshotAdded(self, session):
""" Notify the widget of the given session about an added snapshot.
"""
for item in self.session_list.findItems('', Qt.MatchRegExp):
widget = self.session_list.itemWidget(item)
if widget:
if widget.getSession() is session:
widget.addAllSnapshots()
def setSelectedSID(self, SID):
""" Sets the list widgets selection to the session with SID.
"""
numItems = self.session_list.count()
for idx in range(numItems):
buff = self.session_list.model().index(idx, 0)
_sid = buff.data(Qt.UserRole)
if _sid == SID:
self.session_list.setCurrentIndex(buff)
def getSelectedSessions(self):
""" Return the currently selected sessions as a list.
"""
ret = []
for item in self.session_list.selectedItems():
sid = item.data(Qt.UserRole)
sess = self.displayed_sessions[sid]
if sess:
ret.append(sess)
return ret
def _onCreate(self):
list = self.mainWindow.hostManager.getActiveHostList() # type: list
if len(list) == 0:
self.controller.justCreateNewSession()
return
list.insert(0, ["local", "Local Host"])
ret = askFromList(self, list, "Select Host for new Session",
"You have configured remote Hosts.\nPlease select where to create the Session.")
if ret:
if ret == "local":
self.controller.justCreateNewSession()
else:
remote = self.mainWindow.hostManager.getHostDataById(ret)
self.controller.justCreateNewSession(remote)
return
def onSelectionChanged(self):
""" Called when the list selection changed.
"""
if len(self.session_list.selectedItems()) == 1:
sid = self.getSelectedSessions()[0].getSessionId()
self.default_actions.changeSession(newSID=sid)
def getFilterState(self):
""" Return the text of the currently filtered state.
"""
return str(self.status_combo_box.currentText())
def setController(self, controller):
""" Set the controller and connect actions to contoller.
"""
self.controller = controller
self.controller.setSessionGui(self)
self.status_combo_box.currentIndexChanged.connect(
self.controller.filterState)
self.session_list.itemSelectionChanged.connect(
self.onSelectionChanged)
Barista/gui/main_window/docks/session/session_controller.py 0000664 ڊ0011117 00000030216 13265130053 024553 0 ustar klemms o0cvpr from collections import OrderedDict
from PyQt5.Qt import QObject
from PyQt5.QtCore import pyqtSignal, Qt
from backend.barista.constraints.session_run.training import checkMinimumTrainingRequirements
from backend.barista.session.client_session import ClientSession
from backend.barista.session.session import State
from backend.barista.session.session_pool import SessionPool
from backend.barista.utils.logger import Log
from backend.barista.utils.logger import LogCaller
class SessionController(QObject, LogCaller):
""" View Controller for DockElementSessions.
This controller handles events from the sessions widget and dispatches
sessions when a user requests to train his configured net.
"""
poolSignal = pyqtSignal(object)
sessionSelectionChanged = pyqtSignal(object)
sessionStateRefreshed = pyqtSignal(object)
def __init__(self):
super(SessionController, self).__init__()
self.project = None
self.caller_id = None
self.plotter = None
self.sessionGui = None
self.wPlotter = None
# self.activeSession = None # stores the session that is currently worked on
def setProject(self, project):
self.project = project
if project:
for sid, session in project.getSessions().iteritems():
self.sessionConnects(session)
session.snapshotAdded.connect(lambda: self.wPlotter.updatePlotter(self.project.getSessions()))
self.filterState()
pool = SessionPool()
self.poolSignal.connect(lambda: self.wPlotter.updatePlotter(self.project.getSessions()))
pool.activate(lambda: self.poolSignal.emit('hey'))
self.updateWeightPlotter()
project.activeSessionChanged.connect(self.refreshAccordingToActiveSession)
def sessionConnects(self, session):
""" Connects different signals to the given session. """
# Without the QueuedConnection a running remote session can't communicate with the server
session.stateChanged.connect(
lambda state=None, session=session: self.sessionStateChanged(session, state), Qt.QueuedConnection)
session.iterationChanged.connect(
lambda session=session: self.sessionIterationChanged(session), Qt.QueuedConnection)
session.snapshotAdded.connect(
lambda _, session=session: self.sessionSnapshotAdded(session), Qt.QueuedConnection)
def sessionStateChanged(self, session, state=None):
""" Handle state-change of the given session. """
if not session.isRemote():
session.updateMaxIterFromStateDict()
self.sessionIterationChanged(session)
self.sessionGui.sessionStateChanged(session, state)
self.refreshAccordingToActiveSession()
if session.getState() == State.RUNNING:
self.displaySession(session.getSessionId())
def refreshAccordingToActiveSession(self):
""" Refresh docks, text editor and node editor according to the active session. """
session = self.project.getActiveSession()
state = session.getState()
# emit signal to handle state change in view_manager
self.sessionStateRefreshed.emit(state)
def sessionIterationChanged(self, session):
""" Handle iteration-change of the given session. """
self.sessionGui.sessionIterationChanged(session)
def sessionSnapshotAdded(self, session):
""" Handle new snapshot for a given session. """
self.sessionGui.sessionSnapshotAdded(session)
def setPlotterGui(self, plotter):
self.plotter = plotter
def setSessionGui(self, sessionGui):
self.sessionGui = sessionGui
def setActiveSID(self, sessionID):
"""Sets the active sessionID.
If a valid sessionID is provided, this method sets the session active
and updates all GUI elements accordingly. A sessionID is valid if a
session item does exist. Session states and connectivity are not
checked."""
if sessionID in self.project.getValidSIDs():
self.filterState()
self.sessionSelectionChanged.emit(sessionID)
self.sessionGui.setSelectedSID(sessionID) # update session list
self.updateWeightPlotter()
def setWeightPlotter(self, wPlotter):
self.wPlotter = wPlotter
def justCreateNewSession(self, remote=None):
""" Creates a new session and returns its session id.
"""
if remote is not None:
sessionID = self.project.createRemoteSession(remote,
state_dictionary=self.sessionGui.mainWindow.networkManager.getStateDictionary())
else:
sessionID = self.project.createSession(
state_dictionary=self.sessionGui.mainWindow.networkManager.getStateDictionary())
self.filterState()
if sessionID is not None:
session = self.project.getSession(sessionID)
self.sessionConnects(session)
self.sessionIterationChanged(session)
self.sessionGui.mainWindow.actions.checkSessionState(session, showWarnings=False)
self.setActiveSID(sessionID)
return sessionID
def loadRemoteSession(self, remote, sessionUID):
sessionID = self.project.loadRemoteSession(remote, sessionUID)
self.filterState()
if sessionID is not None:
session = self.project.getSession(sessionID)
session.fetchParserData()
self.sessionConnects(session)
self.setActiveSID(sessionID)
def createNewSession(self, caffemodel=None):
""" Create a new session and start the training.
Create a new session, add new parser to plotter and create a new
session widget for the session
"""
if checkMinimumTrainingRequirements(self.project, self.sessionGui):
sessionID = self.project.createSession(
state_dictionary=self.sessionGui.mainWindow.networkManager.getStateDictionary())
if sessionID:
session = self.project.getSessions[sessionID]
session.snapshotAdded.connect(lambda: self.wPlotter.updatePlotter(self.project.getSessions()))
self.updateWeightPlotter()
# Since this is not set at startup, the parsing has already been done, thus the session should send signals
session.setParserInitialized()
session.start(caffemodel=caffemodel)
self.filterState()
else:
Log.error('Failed to create session!', self.getCallerId())
def displaySession(self, sessionID):
""" Displays a session in the plotter.
"""
session = self.project.getSession(sessionID)
if self.plotter and session is not None:
parser = session.getParser()
if parser:
# add plot (True -> plotOnUpdate = realtime)
self.plotter.putLog(str(session.getLogId()), parser, True, str(session.getLogFileName(True)))
def updateWeightPlotter(self, input=None):
""" Updates Plotter"""
self.wPlotter.updatePlotter(self.project.getSessions())
def createNewSessionFromSnapshot(self, solverstate, old_session):
""" Create a new session as a clone of the given session, start with the
snapshot state.
"""
if old_session.isRemote():
sessionID = self.project.cloneRemoteSession(solverstate, old_session)
else:
sessionID = self.project.cloneSession(solverstate, old_session)
session = self.project.getSession(sessionID)
if session:
session.setState(State.PAUSED)
self.sessionConnects(session)
session.snapshotAdded.connect(lambda: self.wPlotter.updatePlotter(self.project.getSessions()))
self.setActiveSID(sessionID)
else:
cid = None
if old_session:
cid = old_session.getCallerId()
else:
cid = self.getCallerId()
Log.error('Failed to clone session!', cid)
def closeSession(self, remoteSession):
""" Closes the given remote session and makes sure that a new valid session will
be selected. If the last session is deleted, a new one will be created
For further in-method documentation see removeSessions() below
"""
if not isinstance(remoteSession, ClientSession):
return
remoteSession.pause()
sessionID = remoteSession.getSessionId()
validSIDs = [s for s in self.project.getValidSIDs() if not s == sessionID]
if len(validSIDs) == 0:
sid = self.justCreateNewSession()
else:
sid = validSIDs[-1]
self.project.setActiveSID(sid)
self.plotter.removeLog(str(remoteSession.getLogId()),
str(remoteSession.getLogFileName(True)))
self.project.closeSession(remoteSession)
self.wPlotter.updatePlotter(self.project.getSessions())
self.filterState()
self.sessionGui.mainWindow.actions.loadSessionState(sid)
self.sessionGui.setSelectedSID(sid)
def removeSessions(self, sessions):
""" Delete the given sessions and make sure that a new valid session will
be selected. If the last session is deleted, we create a new one, to keep
a sane project state.
"""
sessionIDs = [s.getSessionId() for s in sessions]
# Get all session ids that remain valid after the sessions have been deleted.
validSIDs = [s for s in self.project.getValidSIDs() if s not in sessionIDs]
# If we delete all valid sessions, we create a new one, because the project
# shouldn't exist without a valid session.
if len(validSIDs) == 0:
sid = self.justCreateNewSession()
else:
sid = validSIDs[-1]
# Now we select a new session from the remaining valid sessions, to prevent
# a conflict when deleting.
self.project.setActiveSID(sid)
# Remove the session.
for session in sessions:
self.plotter.removeLog(str(session.getLogId()),
str(session.getLogFileName(True)))
self.project.removeSessions(sessions)
# Update the weight plotter.
self.wPlotter.updatePlotter(self.project.getSessions())
# Update the session list widget, to show the remaining session items.
self.filterState()
# And finally we call the default action to load the newly selected session.
# TODO: this should eventually be called from sessionGui.setSelectedSID()
# anyway, however without this line it doesn't seem to work.
self.sessionGui.mainWindow.actions.loadSessionState(sid)
# And select the newly selected session from the list widgets.
self.sessionGui.setSelectedSID(sid)
# TODO: Are the deleted sessions garbage collected at this point or do we
# have a memory leak?
def filterState(self):
""" Filter the sessions for the state that has been selected in the GUI.
"""
filtered = OrderedDict()
state = self.sessionGui.getFilterState()
if state == 'ALL':
filtered = self.project.getSessions()
else:
for sid, session in self.project.getSessions().iteritems():
sesstate = session.getState()
if ((state == 'RUNNING' and sesstate is State.RUNNING) or
(state == 'FAILED' and sesstate is State.FAILED) or
(state == 'FINISHED' and sesstate is State.FINISHED) or
(state == 'PAUSED' and sesstate is State.PAUSED) or
(state == 'INVALID' and sesstate is State.INVALID) or
(state == 'WAITING' and sesstate is State.WAITING)):
filtered[sid] = session
self.sessionGui.showSessions(filtered)
# LogCaller
def getCallerId(self):
""" Return the unique caller id for this session
"""
if self.caller_id is None:
self.caller_id = Log.getCallerId('SessionController')
return self.caller_id
def resetSession(self, session):
session.reset()
self.sessionStateChanged(session)
Barista/gui/main_window/docks/session/session_item.py 0000664 ڊ0011117 00000037735 13265130053 023343 0 ustar klemms o0cvpr import os
from PyQt5.QtCore import QPoint, Qt
from PyQt5.QtGui import QPixmap, QIcon, QCursor
from PyQt5.QtWidgets import (
QWidget,
QPushButton,
QLabel,
QMessageBox,
QVBoxLayout,
QHBoxLayout,
QProgressBar,
QMenu,
QAction)
from backend.barista.session.session_utils import State
from backend.barista.session.client_session import ClientSession
from backend.barista import caffe_versions
from backend.caffe import loader
class SessionWidget(QWidget):
""" A instance of this widget represents an entry for a session in the list
of session within the running jobs dock element.
"""
#TODO: add an item to select a session as visible/editable i.e. 'active'?
def __init__(self, session, controller, parent=None):
super(SessionWidget, self).__init__(parent)
self.session = session
self.controller = controller
mainLayout = QVBoxLayout(self)
buttonLayout = QHBoxLayout()
progressLayout = QHBoxLayout()
remoteLayout = QHBoxLayout()
# Add a network label if this is a remote session.
if isinstance(self.session, ClientSession):
self.buttonClose = QPushButton()
self.buttonClose.setObjectName("sessionBtn")
self.buttonClose.setToolTip("Close this session. (Does not remove the session from the server)")
closeIcon = QPixmap('resources/unlink.png')
self.buttonClose.setIcon(QIcon(closeIcon))
self.buttonClose.setFixedSize(30, 20)
self.buttonClose.clicked.connect(self.onClose)
remoteLayout.addWidget(self.buttonClose)
self.networkLabel = QLabel()
remote = self.session.remote
if len(remote) > 2: # if remote host has a name, display this name
self.networkLabel.setText(remote[2] + ":" + str(remote[1]))
else:
self.networkLabel.setText(remote[0] + ":" + str(remote[1]))
self.networkLabel.setAlignment(Qt.AlignLeft | Qt.AlignVCenter)
remoteLayout.addWidget(self.networkLabel)
mainLayout.addLayout(remoteLayout)
# Create a label for the session id.
sessionIdLabel = QLabel(str(session.getSessionId()))
sessionIdLabel.setObjectName("sessionId")
sessionIdLabel.setFixedSize(30, 30)
sessionIdLabel.setAlignment(Qt.AlignCenter)
buttonLayout.addWidget(sessionIdLabel)
# Create session status label.
self.statusLabel = QLabel()
self.statusLabel.setObjectName("sessionStatus")
self.statusLabel.setFixedHeight(30)
self.statusLabel.setMinimumWidth(100)
self.statusLabel.setAlignment(Qt.AlignCenter)
buttonLayout.addWidget(self.statusLabel)
# Create snapshot button.
self.snapshotBtn = QPushButton()
self.snapshotBtn.setObjectName("sessionBtn")
snapshotIcon = QPixmap('resources/snap.png')
self.snapshotBtn.setIcon(QIcon(snapshotIcon))
self.snapshotBtn.setToolTip("Take a snapshot")
self.snapshotBtn.setFixedSize(30, 30)
self.snapshotBtn.clicked.connect(self.onSnapshot)
buttonLayout.addWidget(self.snapshotBtn)
# Create delete button.
self.deleteBtn = QPushButton()
self.deleteBtn.setObjectName("sessionBtn")
deleteIcon = QPixmap('resources/trash.png')
self.deleteBtn.setIcon(QIcon(deleteIcon))
self.deleteBtn.setToolTip("Delete Session")
self.deleteBtn.setFixedSize(30, 30)
self.deleteBtn.clicked.connect(self.onDelete)
buttonLayout.addWidget(self.deleteBtn)
# Create more button.
self.moreBtn = QPushButton()
self.moreBtn.setObjectName("sessionBtn")
moreIcon = QPixmap('resources/more.png')
self.moreBtn.setIcon(QIcon(moreIcon))
self.moreBtn.setToolTip("Show more actions")
self.moreBtn.setFixedSize(30, 30)
# Add actions to the more button menu.
moreBtnMenu = QMenu(self.moreBtn)
self.cloneSessionMenu = moreBtnMenu.addMenu("Clone Session")
self.moreBtn.setMenu(moreBtnMenu)
resetAct = QAction("Reset", moreBtnMenu)
resetAct.triggered.connect(self.onReset)
moreBtnMenu.addAction(resetAct)
self.addAllSnapshots()
buttonLayout.addWidget(self.moreBtn)
# Add a stretch, so that the horizontal items are left aligned.
mainLayout.addLayout(buttonLayout)
# Create start button.
self.startBtn = QPushButton()
self.startBtn.setObjectName("sessionBtn")
startIcon = QPixmap('resources/start.png')
self.startBtn.setIcon(QIcon(startIcon))
self.startBtn.setToolTip("Start Training")
self.startBtn.setFixedSize(30, 30)
self.startBtn.clicked.connect(self.onStart)
self.startBtn.setEnabled(False)
progressLayout.addWidget(self.startBtn)
# Create pause button.
self.pauseBtn = QPushButton()
self.pauseBtn.setObjectName("sessionBtn")
pauseIcon = QPixmap('resources/pause.png')
self.pauseBtn.setIcon(QIcon(pauseIcon))
self.pauseBtn.setToolTip("Pause Training")
self.pauseBtn.setFixedSize(30, 30)
self.pauseBtn.clicked.connect(self.onPause)
self.pauseBtn.setEnabled(False)
progressLayout.addWidget(self.pauseBtn)
# Add a progress bar.
self.progress = QProgressBar()
self.progress.setTextVisible(True)
self.sessionIterationChanged()
progressLayout.addWidget(self.progress)
mainLayout.addLayout(progressLayout)
#Add text label for invalid sessions
self.invalidLabel = QLabel()
self.invalidLabel.setObjectName("invalidMsg")
self.invalidLabel.setFixedHeight(30)
self.invalidLabel.setMinimumWidth(80)
self.invalidLabel.setAlignment(Qt.AlignCenter)
self.invalidLabel.setEnabled(False)
progressLayout.addWidget(self.invalidLabel)
# Update the current session state.
self.sessionStateChanged(self.session.getState())
def getSession(self):
""" Return the session of this widget.
"""
return self.session
def getSID(self):
""" Returns the session ID related to this session item.
"""
return self.session.getSessionId()
def showDialog(self, text, info, title, location=None, buttons=None):
""" Show a message box to the user.
Return the user decision.
"""
msg = QMessageBox()
msg.setIcon(QMessageBox.Information)
msg.setText(text)
msg.setInformativeText(info)
msg.setWindowTitle(title)
if buttons is not None:
try:
msg.setStandardButtons(buttons)
except TypeError:
# if something wrong is passed to the buttons Parameter, show the standard message.
msg.setStandardButtons(QMessageBox.Ok | QMessageBox.Cancel)
else:
msg.setStandardButtons(QMessageBox.Ok | QMessageBox.Cancel)
if location is None:
location = QCursor.pos()
msg.move(location - QPoint(msg.sizeHint().width() / 2,
msg.sizeHint().height() / 2))
return msg.exec_()
def sessionIterationChanged(self):
""" The sessions iteration changed.
"""
iteration = self.session.getIteration()
maxIteration = self.session.getMaxIteration()
try:
i = float(iteration) * 100.0
m = float(maxIteration)
self.progress.setValue(i/m)
except Exception:
pass
text = str(int(iteration))+' / '+str(maxIteration)
self.progress.setFormat(text)
def addAllSnapshots(self):
""" Add all snapshots to the clone session submenu.
"""
snapshots = self.session.getSnapshots()
self.cloneSessionMenu.clear()
for iter_num in sorted(snapshots.keys()):
self.addSnapshot(snapshots[iter_num])
snapshots = self.session.getSnapshots()
snapshotCount = len(snapshots)
if snapshotCount == 0:
self.cloneSessionMenu.setEnabled(False)
else:
self.cloneSessionMenu.setEnabled(True)
def addSnapshot(self, snapshot):
""" Add a snapshot to the snapshot menu.
"""
snapAct = QAction(os.path.basename(snapshot), self.cloneSessionMenu)
snapAct.triggered.connect(lambda: self.createFromSnapshot(snapshot))
self.cloneSessionMenu.addAction(snapAct)
def createFromSnapshot(self, snapshot=None):
""" Called when the snapshot tool button was clicked or when a snapshot
was selected in the snapshot dropdown.
"""
info = "A new session will be created from the snapshot " + snapshot
if self.showDialog("Do you want to continue from this snapshot?",
info, "") == QMessageBox.Ok:
self.controller.createNewSessionFromSnapshot(snapshot, self.session)
def sessionStateChanged(self, state=None):
""" Called when the session state changed.
"""
if state is None:
state = self.session.getState()
if state is State.WAITING:
self.__buttons_wait()
self.__progress_show()
if self.session.getPretrainedWeights():
self.setStatus('Pre-trained', '#8000ff')
self.statusLabel.setToolTip('Ready to start with pre-trained weights!')
else:
self.setStatus('Ready', 'blue')
self.statusLabel.setToolTip('Ready to train!')
elif state is State.RUNNING:
self.__buttons_run()
self.__progress_show()
self.setStatus('Running', 'green')
self.statusLabel.setToolTip('Currently training..')
elif state is State.PAUSED:
self.__buttons_wait()
self.__progress_show()
self.setStatus('Paused', 'orange')
self.statusLabel.setToolTip('Training paused.')
elif state is State.FINISHED:
self.__buttons_disable()
self.__progress_show()
self.setStatus('Finished', 'gray')
self.statusLabel.setToolTip('Finished, training completed.')
elif state is State.FAILED:
self.__buttons_disable()
self.__progress_show()
self.setStatus('Failed', 'red')
self.statusLabel.setToolTip('Something went wrong.')
elif state is State.UNDEFINED:
self.__buttons_disable()
self.__progress_show()
self.setStatus('Undefined', 'gray')
self.statusLabel.setToolTip('')
elif state is State.INVALID:
self.__buttons_disable()
self.setStatus('Invalid', 'gray')
self.statusLabel.setToolTip('Session is not ready to train.')
self.__progress_hide()
errorList = self.session.getErrorList()
'''
ErrorList is a list of all constraints that are broken for this session. Each tuple contains
a full error message displayed in the tooltip (index 0) and a key word to be shown in the status box.
'''
if errorList is not None and len(errorList) > 0:
self.invalidLabel.setToolTip(errorList[0][0])
self.setInvalidMsg((errorList[0])[1])
elif state is State.NOTCONNECTED:
self.__buttons_disable()
self.__progress_hide()
self.setStatus('Invalid', 'gray')
self.setInvalidMsg('No connection to host found.')
self.invalidLabel.setToolTip('')
self.statusLabel.adjustSize()
def setStatus(self, text, color='gray'):
""" Set the style of the status label.
"""
self.statusLabel.setText(text)
# If we just want to overwrite single properties of the stylesheet, just
# append them to the current stylesheet.
self.statusLabel.setStyleSheet(self.statusLabel.styleSheet() + " QLabel { background-color: "+color+"; border-color: "+color+" }" )
def setInvalidMsg(self, text, color='gray'):
self.invalidLabel.setText(text)
self.invalidLabel.setStyleSheet(
self.invalidLabel.styleSheet() + " QLabel { background-color: " + color + "; border-color: " + color + " }")
def __buttons_run(self):
self.startBtn.setEnabled(False)
self.startBtn.hide()
self.pauseBtn.setEnabled(True)
self.pauseBtn.show()
self.snapshotBtn.setEnabled(True)
def __buttons_wait(self):
self.startBtn.setEnabled(True)
self.startBtn.show()
self.pauseBtn.setEnabled(False)
self.pauseBtn.hide()
self.snapshotBtn.setEnabled(False)
def __buttons_disable(self):
self.startBtn.setEnabled(False)
self.startBtn.show()
self.pauseBtn.setEnabled(False)
self.pauseBtn.hide()
self.snapshotBtn.setEnabled(False)
def __progress_hide(self):
self.startBtn.setEnabled(False)
self.startBtn.show()
self.progress.setEnabled(False)
self.progress.hide()
self.invalidLabel.setEnabled(True)
self.invalidLabel.show()
def __progress_show(self):
self.progress.setEnabled(True)
self.progress.show()
self.invalidLabel.setEnabled(False)
self.invalidLabel.hide()
def onStart(self):
""" Start button was clicked.
"""
if self.session.getState() is State.WAITING:
self.session.start(caffemodel=self.session.getPretrainedWeights())
elif self.session.getState() is State.PAUSED:
self.session.proceed()
def onPause(self):
self.session.pause()
def onSnapshot(self):
self.session.snapshot()
def onDelete(self):
""" Delete button was clicked.
"""
sid = self.getSID()
if self.session.getState() == State.RUNNING:
confirm = self.showDialog("The selected session is still running.\nDo you want to stop it "
"and delete Session {} anyway?".format(sid), "",
"Delete Session") == QMessageBox.Ok
else:
confirm = self.showDialog("Do you want to delete Session {}?".format(sid), "",
"Delete Session") == QMessageBox.Ok
if confirm:
self.controller.removeSessions((self.session,))
def onClose(self):
""" Close button was clicked.
"""
sid = self.getSID()
if self.session.getState() == State.RUNNING:
confirm = self.showDialog("The selected session is still running.\nDo you want to stop it "
"and close the connection anyway?".format(sid), "",
"Close Session") == QMessageBox.Ok
else:
confirm = True
if confirm:
self.controller.closeSession(self.session)
def onReset(self):
""" Reset button was clicked.
"""
sid = self.getSID()
if self.session.getState() == State.RUNNING:
confirm = self.showDialog(
"The selected session is still running.\nDo you want to stop and reset Session {} anyway?\n"
"All progress (including logs) will be lost!".format(sid), "", "Reset Session") == QMessageBox.Ok
elif self.session.getState() == State.PAUSED:
confirm = self.showDialog(
"The selected session was trained before.\nDo you want to reset Session {} anyway?\n"
"All progress (including logs) will be lost!".format(sid), "", "Reset Session") == QMessageBox.Ok
else:
confirm = self.showDialog("Do you want to reset Session {}?".format(sid), "",
"Reset Session") == QMessageBox.Ok
if confirm:
# Remove all snapshots from the context menu
for action in self.cloneSessionMenu.actions():
self.cloneSessionMenu.removeAction(action)
self.cloneSessionMenu.setDisabled(True)
self.controller.resetSession(self.session)
Barista/gui/main_window/docks/weight_visualization/__init__.py 0000664 ڊ0011117 00000000000 13265130053 025135 0 ustar klemms o0cvpr Barista/gui/main_window/docks/weight_visualization/dock_weight_visualization.py 0000664 ڊ0011117 00000035732 13265130053 030672 0 ustar klemms o0cvpr import os
from PyQt5.QtWidgets import (
QVBoxLayout,
QHBoxLayout,
QComboBox,
QMessageBox,
QWidget,
QPushButton,
QFileDialog,
QSizePolicy
)
from backend.barista.utils.logger import Log
from backend.barista.utils.logger import LogCaller
from backend.caffe import loader
from backend.barista.session.session import Session
from backend.barista.session.client_session import ClientSession
from gui.main_window.docks.dock import DockElement
from gui.main_window.docks.weight_visualization.image_canvas import ImageCanvas
from gui.main_window.docks.weight_visualization.weights import calculateConvWeights
from gui.main_window.docks.weight_visualization.weights import loadNetParameter
class DockElementWeightPlotter(DockElement, LogCaller):
""" This dock element displays the weights of a layer in a selected
session's snapshot. Currently only convolutional layer are supported.
The canvas renders the convolutional filter's weights in a grid layout.
The selection of sessions, snapshots and layers is possible by selecting
them in one of the three comboboxes.
"""
ALLOWED_LAYERTYPES = {"Convolution"}
class SelectComboBox(QComboBox):
""" This class is a template for the combobox needed to choose the snapshots, layers, and session"""
# emptyMessage gets printed on the Combobox when no element has been added to it.
def __init__(self, emptyMessage):
QComboBox.__init__(self)
# Change layout to neatly fit the text
self.setSizePolicy(QSizePolicy.Maximum, QSizePolicy.Minimum)
self.setSizeAdjustPolicy(self.AdjustToContents)
self.__emptyMessage = emptyMessage
def replaceItems(self, idList):
""" Replace all items with the current List"""
self.clear()
if idList is None or idList == []:
self.setEnabled(False)
self.addItems([self.__emptyMessage])
self.setCurrentText(self.__emptyMessage)
else:
self.setEnabled(True)
QComboBox.addItems(self, idList)
def __init__(self, mainWindow, title):
DockElement.__init__(self, mainWindow, title)
self.caller_id = None
self.__setupGui()
# Initialize the comboboxes needed
self.sessionComboBox.activated['QString'].connect(self.__setCurrentSession)
self.snapshotComboBox.activated['QString'].connect(self.__setCurrentSnapshot)
self.layerComboBox.activated['QString'].connect(self.__setCurrentLayer)
self.__currentSessionId = None
self.__currentSnapshotId = None
self.__currentLayerName = None
self.__currentNet = None
self.__sessionDict = {}
# Stores the snapshots which has been selected by the user, to load them faster the next time they are needed.
self.__alreadyOpenSnapshots = {}
def __setupGui(self):
"""Add layouts and populate them with widgets"""
self.mainWidget = QWidget()
self.mainLayout = QVBoxLayout(self.mainWidget)
self.setWidget(self.mainWidget)
# Create the comboboxes to select a visualization.
self.selectionLayout = QHBoxLayout()
self.mainLayout.addLayout(self.selectionLayout)
self.sessionComboBox = DockElementWeightPlotter.SelectComboBox("No sessions")
# Set the sessionComboBox's size policy to force it to display the whole name.
self.sessionComboBox.setSizePolicy(QSizePolicy.Minimum, QSizePolicy.Minimum)
self.selectionLayout.addWidget(self.sessionComboBox)
self.snapshotComboBox = DockElementWeightPlotter.SelectComboBox("No snapshots")
self.selectionLayout.addWidget(self.snapshotComboBox)
self.layerComboBox = DockElementWeightPlotter.SelectComboBox("No layers")
self.selectionLayout.addWidget(self.layerComboBox)
# Create the 'Save as Image..' button right aligned and connect it's signal.
self.selectionLayout.addStretch()
self.saveImageButton = QPushButton("Save as Image..")
self.saveImageButton.clicked.connect(self.__saveImage)
self.selectionLayout.addWidget(self.saveImageButton)
# Create the canvas to display the plot.
self.canvasWidget = ImageCanvas()
self.mainLayout.addWidget(self.canvasWidget)
def __setCurrentSession(self, sid):
if self.__currentSessionId == sid:
return
self.__currentSessionId = sid
self.__updateSnapshotList()
self.__updateLayerList()
self.__updateCanvas()
def __setCurrentSnapshot(self, sid):
if self.__currentSnapshotId == sid:
return
self.__currentSnapshotId = sid
self.__updateCanvas()
def __setCurrentLayer(self, lid):
if self.__currentLayerName == lid:
return
self.__currentLayerName = lid
self.__updateCanvas()
def __updateCanvas(self):
""" Updates the current shown picture with the current settings """
# Load model
self.__currentNet = self.__getNetwork()
if self.__currentNet:
self.saveImageButton.setEnabled(True)
image = calculateConvWeights(self.__currentNet, self.__currentLayerName)
if image is not None:
self.canvasWidget.showImage(image)
self.canvasWidget.show()
else:
# Hide canvas when no snapshot, session or layer can be chosen
self.canvasWidget.hide()
self.saveImageButton.setEnabled(False)
def __getNetwork(self, sess_id=None, snap_id=None):
""" Return the caffe network of the current session and snapshot.
"""
if sess_id is None:
sess_id = self.__currentSessionId
if snap_id is None:
snap_id = self.__currentSnapshotId
if sess_id and snap_id:
# Creates a Dictionary of Sessions Ids, which contains Snapshot Ids
# which point to Layer Id which point to
# already opened Networks.
net = None
if sess_id in self.__alreadyOpenSnapshots:
session_snapshots = self.__alreadyOpenSnapshots[sess_id]
if snap_id in session_snapshots:
net = session_snapshots[snap_id]
if net:
# cached net found
return net
else:
# snapshot was accessed for the first time
# create and cache the net
session = self.__sessionDict[sess_id]
snapName = snap_id.replace('solverstate', 'caffemodel')
if isinstance(session, ClientSession):
net = session.loadNetParameter(snapName)
if net is None:
return
else:
snapshotPath = session.getSnapshotDirectory()
snapshotPath = str(os.path.join(snapshotPath, snapName))
if not os.path.exists(snapshotPath):
Log.error('Snapshot file '+snapshotPath+' does not exist!', self.getCallerId())
return
net = loadNetParameter(snapshotPath)
if net is not None:
if sess_id not in self.__alreadyOpenSnapshots.keys():
self.__alreadyOpenSnapshots[sess_id] = {snap_id: net}
else:
self.__alreadyOpenSnapshots[sess_id][snap_id] = net
return net
else:
# Show a warning message
Log.error('The hdf5 snapshot format is not supported for the weight visualization! '
'This can be changed by setting the snapshot_format parameter in the solver properties.', self.getCallerId())
def updatePlotter(self, sessionDict):
""" Updates the comboboxes and redraws the weight image.
"""
# convert ids to string
self.__sessionDict = {
"Session " + str(sid):
session for sid, session in sessionDict.iteritems()
}
self.__updateSessionsList()
self.__updateSnapshotList()
self.__updateLayerList()
self.__updateCanvas()
def __updateSessionsList(self):
""" Should be called, if the Id-List changes. Updates the session
Combobox.
"""
# get a ordered List from the session-dictionary
idList = map(
lambda tuple: tuple[0],
sorted(self.__sessionDict.items(),
key=lambda id: id[1].getSessionId()))
# updates the sessionCombobox with the current sessions
self.sessionComboBox.replaceItems(idList)
# Tries to apply the last selected session, if it exist (could be
# deleted). If not, select the last one(assume sorted)
if self.__currentSessionId is None or self.__currentSessionId not in idList:
if idList == []:
self.__currentSessionId = None
else:
self.__currentSessionId = idList[-1]
self.sessionComboBox.setCurrentText(self.__currentSessionId)
def __updateSnapshotList(self):
""" Should be called, if the Id-List changes. Updates the snapshot
Combobox.
"""
# assert that there is a current session to show
if self.__currentSessionId is not None:
snapshotDict = self.__sessionDict[self.__currentSessionId].getSnapshots()
snapshotsIdList = map(
lambda tuple: tuple[1],
sorted(snapshotDict.items(), key=lambda id: id[0]))
else:
snapshotsIdList = []
# update the snapshotCombobox with the current snapshots
self.snapshotComboBox.replaceItems(snapshotsIdList)
# Select the last snapshot, or None if there are none
if not snapshotsIdList == []:
self.__currentSnapshotId = snapshotsIdList[-1]
else:
self.__currentSnapshotId = None
self.snapshotComboBox.setCurrentText(self.__currentSnapshotId)
def __updateLayerList(self):
""" Update the layer list with available layers found in the net
description.
"""
# getLayers
if self.__currentSessionId is not None:
session = self.__sessionDict[self.__currentSessionId]
if isinstance(session, ClientSession):
netInternal = session.loadInternalNetFile()
currentNetwork = loader.loadNet(netInternal)
layerNames = map(
lambda layer: layer["parameters"]["name"],
filter(
lambda layer: layer["type"].name() in self.ALLOWED_LAYERTYPES,
currentNetwork["layers"].values()
)
)
layerNameList = sorted(layerNames)
else:
try:
currentNetworkPath = session.getInternalNetFile()
file = open(currentNetworkPath, 'r')
currentNetwork = loader.loadNet(file.read())
# get all the names of the layers, which match the desired type
layerNames = map(
lambda layer: layer["parameters"]["name"],
filter(
lambda layer: layer["type"].name() in self.ALLOWED_LAYERTYPES,
currentNetwork["layers"].values()
)
)
layerNameList = sorted(layerNames)
except IOError:
callerId = Log.getCallerId('weight-plotter')
Log.error("Could not open the network of this session.", callerId)
layerNameList =[]
else:
layerNameList =[]
# updates the layer Combobox with the current layers
self.layerComboBox.replaceItems(layerNameList)
if self.__currentLayerName is None or self.__currentLayerName not in layerNameList:
if not layerNameList == []:
self.__currentLayerName = layerNameList[-1]
else:
self.__currentLayerName = None
self.layerComboBox.setCurrentText(self.__currentLayerName)
def __saveImage(self):
""" Gives the user the opportunity to save the image on disk.
The user can choose from allowed extension, and is alerted if the entered extension is not valid.
If no extension is entered by the user, the extension is set to png.
"""
allowedExtensions = [".png", ".bmp", ".jpeg", ".jpg"]
allowedAsString = str(reduce(lambda x, y: x + y, map(lambda x: " *" + x, allowedExtensions)))
callerId = self.getCallerId()
try:
filename = ""
# While the entered extension does not matches the allowd extensions
while not self.__validateFilename(filename, allowedExtensions):
fileDialog = QFileDialog()
fileDialog.setDefaultSuffix("png")
filenameArray = fileDialog.getSaveFileName(
self,
"Save File",
filename,
allowedAsString
)
filename = filenameArray[0]
if filename != "":
_, extension = os.path.splitext(filename)
# If no extension has been entered, append .png
if extension == "":
filename += ".png"
else:
# If user clicks on abort, leave the loop
break
# Show an alert message, when an unknown extension has been entered
if not self.__validateFilename(filename, allowedExtensions):
msg = QMessageBox()
msg.setIcon(QMessageBox.Warning)
msg.setWindowTitle("Warning")
msg.setText("Please enter a valid extension:\n"
+ allowedAsString)
msg.exec_()
if filename != "":
if self.__currentNet:
image = calculateConvWeights(self.__currentNet, self.__currentLayerName)
self.canvasWidget.saveImage(image, filename)
Log.log("Saved image under " + filename, callerId)
else:
Log.error("Try to save image of weights without a network being loaded.", callerId)
except Exception as e:
Log.error("Saving the file failed. " + str(e), callerId)
# LogCaller
def getCallerId(self):
""" Return the unique caller id for this session
"""
if self.caller_id is None:
self.caller_id = Log.getCallerId('weight-plotter')
return self.caller_id
def __validateFilename(self, filename, allowedExtensions):
""" Checks if filename is valid, returns corresponding boolean"""
_, extension = os.path.splitext(filename)
if extension not in allowedExtensions:
return False
else:
return True
Barista/gui/main_window/docks/weight_visualization/image_canvas.py 0000664 ڊ0011117 00000004537 13265130053 026036 0 ustar klemms o0cvpr from PyQt5.QtWidgets import (
QGraphicsView,
QGraphicsScene,
QGraphicsPixmapItem
)
from PyQt5 import QtCore
from PyQt5.QtGui import QPixmap, QImage, QColor
class ImageCanvas(QGraphicsView):
""" This class manages the view of the weights. """
def __init__(self):
QGraphicsView.__init__(self)
self.__scene = QGraphicsScene(self)
self.__item = QGraphicsPixmapItem()
self.__scene.addItem(self.__item)
self.setScene(self.__scene)
self.setTransformationAnchor(QGraphicsView.AnchorUnderMouse)
self.setResizeAnchor(QGraphicsView.AnchorUnderMouse)
# Change resize policy when hidden, to ensure it takes the same space
pol = self.sizePolicy()
pol.setRetainSizeWhenHidden(True)
self.setSizePolicy(pol)
def center(self):
""" Centers the pictures and adjust the zoom-level to maximize the picture."""
pictureRect = QtCore.QRectF(self.__item.pixmap().rect())
self.fitInView(pictureRect, QtCore.Qt.KeepAspectRatio)
def showImage(self, rawImage):
""" Takes a raw Image and show it in the view. """
self.__image = self._getPixImage(rawImage)
self.setDragMode(QGraphicsView.ScrollHandDrag)
self.__item.setPixmap(self.__image)
self.__scene = QGraphicsScene(self)
self.__scene.addItem(self.__item)
self.setScene(self.__scene)
#self.center()
def _getPixImage(self, image):
""" generate a QPixImage from the raw input Image. """
height = image.shape[0]
width = image.shape[1]
qImage = QImage(width, height, QImage.Format_RGB32)
for x in range(0, width):
for y in range(0, height):
val = int(255*image[y][x])
qImage.setPixel(x, y, QColor.fromRgb(val, val, val, 1).rgb())
return QPixmap.fromImage(qImage)
def wheelEvent(self, event):
""" Overrides wheel event to allow zooming. """
mouseDelta = event.angleDelta().y()/120.0
scale = 1.0 + 0.1*mouseDelta
self.scale(scale, scale)
# idea from http://stackoverflow.com/questions/6915106/saving-a-numpy-array-as-an-image-instructions
def saveImage(self, rawImage, filename):
image = self._getPixImage(rawImage)
if not image.save(filename):
raise IOError("Could not save file in the chosen folder.")
Barista/gui/main_window/docks/weight_visualization/run_weights.py 0000664 ڊ0011117 00000001306 13265130053 025746 0 ustar klemms o0cvpr # from https://www.eriksmistad.no/visualizing-learned-features-of-a-caffe-neural-network/
# test code, will be changed
from backend.caffe.path_loader import PathLoader
caffe = PathLoader().importCaffe()
from gui.main_window.docks.weight_visualization import visualize_weights
# Load model
net = caffe.Net('/Users/sandtil/Documents/Developer/pyenv/Barista/test/examples/mnist/sessions/20161221_151232_203/net.prototxt',
'/Users/sandtil/Documents/Developer/pyenv/Barista/test/examples/mnist/sessions/20161221_151232_203/snapshots/lenet_iter_10000.caffemodel',
caffe.TEST)
visualize_weights(net, 'conv1', filename='conv1')
#visualize_weights(net, 'conv2', filename='conv2.png')
Barista/gui/main_window/docks/weight_visualization/weights.py 0000664 ڊ0011117 00000005406 13265130053 025067 0 ustar klemms o0cvpr # from https://www.eriksmistad.no/visualizing-learned-features-of-a-caffe-neural-network/
import numpy as np
def loadNetParameter(caffemodel):
""" Return a NetParameter protocol buffer loaded from the caffemodel.
"""
from backend.caffe.path_loader import PathLoader
proto = PathLoader().importProto()
net = proto.NetParameter()
try:
with open(caffemodel, 'rb') as f:
net.ParseFromString(f.read())
return net
except:
pass
def loadNetParamFromString(paramstring):
from backend.caffe.path_loader import PathLoader
proto = PathLoader().importProto()
net = proto.NetParameter()
try:
net.ParseFromString(paramstring)
return net
except:
pass
def calculateConvWeights(net, layer_name, padding=2):
""" Return a grayscale image array which displays the weights of the
convolutional layer in the net.
"""
for layer in net.layer:
if layer.name == layer_name:
for blob in layer.blobs:
if len(blob.shape.dim) == 4:
# The parameters are a list of [weights, biases]
data = np.copy(blob.data).reshape(blob.shape.dim)
# N is the total number of convolutions
N = blob.shape.dim[0]*blob.shape.dim[1]
# Ensure the resulting image is square
filters_per_row = int(np.ceil(np.sqrt(N)))
# Assume the filters are square
filter_size = blob.shape.dim[2]
# Size of the result image including padding
result_size = filters_per_row*(filter_size + padding) - padding
# Initialize result image to all zeros
result = np.zeros((result_size, result_size))
# Tile the filters into the result image
filter_x = 0
filter_y = 0
for n in range(blob.shape.dim[0]):
for c in range(blob.shape.dim[1]):
if filter_x == filters_per_row:
filter_y += 1
filter_x = 0
for i in range(filter_size):
for j in range(filter_size):
x = filter_y*(filter_size + padding) + i
y = filter_x*(filter_size + padding) + j
result[x, y] = data[n, c, i, j]
filter_x += 1
# Normalize image to 0-1
min = result.min()
max = result.max()
result = (result - min) / (max - min)
return result
Barista/gui/main_window/node_editor/items/__init__.py 0000664 ڊ0011117 00000000000 13265130053 023176 0 ustar klemms o0cvpr Barista/gui/main_window/node_editor/items/connection_item.py 0000664 ڊ0011117 00000020010 13265130053 024617 0 ustar klemms o0cvpr from PyQt5.QtWidgets import QGraphicsPathItem, QMenu
from PyQt5.QtCore import QPointF
from PyQt5.QtGui import QPainterPath, QPainterPathStroker
from gui.main_window.node_editor.items.renderer.connection_item_renderer import ConnectionItemRenderer
from gui.main_window.node_editor.node_editor_constants import Constants
class ConnectionItem(QGraphicsPathItem):
""" A ConnectionItem represents a top-bottom connection in the graphics scene """
def __init__(self, nodeEditor, parent=None):
super(ConnectionItem, self).__init__(parent)
# make selectable
self.setFlag(QGraphicsPathItem.ItemIsSelectable, True)
# draw behind node items
self.setZValue(0.5)
self.__isInPlace = False
self.__nodeEditor = nodeEditor
self.__hidden = False
self.__topConnector = None
self.__bottomConnector = None
self.__start = QPointF(0, 0)
self.__end = QPointF(0, 0)
# create a renderer to separate drawing from logic
self.__renderer = ConnectionItemRenderer(self)
self.__updatePath()
def paint(self, painter, option, widget=None):
self.__renderer.paint(painter)
def shape(self):
stroker = QPainterPathStroker()
stroker.setWidth(Constants.connectionItemSize *3)
return stroker.createStroke(self.path())
def setStart(self, start):
""" Sets the start position of the connection and recalculates the path """
self.__start = start
self.__updatePath()
def setEnd(self, end):
""" Sets the end position of the connection and recalculates the path """
self.__end = end
self.__updatePath()
def getTopConnector(self):
""" Returns the connections top connector """
return self.__topConnector
def getBottomConnector(self):
""" Returns the connections bottom connector """
return self.__bottomConnector
def setConnector(self, connector):
""" Sets the top/bottom connector of the connection. Whether the top or bottom connector gets set,
is decided by information provided by the connector """
if connector.isTopConnector():
self.setTopConnector(connector)
else:
self.setBottomConnector(connector)
def setTopConnector(self, connector):
""" Sets the top connector and updates the connections path """
self.__topConnector = connector
self.__start = connector.scenePos()
self.updateData()
self.__updatePath()
def setBottomConnector(self, connector):
""" Sets the bottom connector and updates the connections path """
self.__bottomConnector = connector
self.__end = connector.scenePos()
self.updateData()
self.__updatePath()
def setHidden(self, hidden):
""" Sets the connection to be hidden/shown and updates the rendering """
self.__hidden = hidden
self.update()
def updateMousePosition(self, pos):
""" Updates the start/end of the connection if the connection is getting dragged (created) """
if self.__topConnector is None:
self.__start = pos
elif self.__bottomConnector is None:
self.__end = pos
self.__updatePath()
def updateData(self):
""" Updates the connections internal in-place and phase variables to update the rendering """
# set in-place
if self.__bottomConnector is not None:
self.__isInPlace = self.__bottomConnector.isInPlace()
else:
self.__isInPlace = False
# update
self.update()
def __updatePath(self):
""" Updates the connections path, using the start and end points """
path = QPainterPath(self.__start)
# the end point is further left than the start point, so draw two half circles and a straight line
if self.__start.x() > self.__end.x():
yDiff = abs(self.__end.y() - self.__start.y())
middleY = (self.__start.y() + self.__end.y()) / 2
# calculate the control points needed for the first half circle
curve1End = QPointF(self.__start.x(), middleY)
curve1CP1 = QPointF(self.__start.x() + yDiff / 2, self.__start.y())
curve1CP2 = QPointF(self.__start.x() + yDiff / 2, middleY)
# calculate the control points needed for the second half circle
curve2Start = QPointF(self.__end.x(), middleY)
curve2CP1 = QPointF(self.__end.x() - yDiff / 2, middleY)
curve2CP2 = QPointF(self.__end.x() - yDiff / 2, self.__end.y())
# draw the first half circle
path.cubicTo(curve1CP1.x(), curve1CP1.y(), curve1CP2.x(), curve1CP2.y(), curve1End.x(), curve1End.y())
# draw the straight line
path.lineTo(curve2Start.x(), curve2Start.y())
# draw the second half circle
path.cubicTo(curve2CP1.x(), curve2CP1.y(), curve2CP2.x(), curve2CP2.y(), self.__end.x(), self.__end.y())
# the start point is further left than the end point, so draw a Bezier curve between both
else:
path.cubicTo((self.__start.x() + self.__end.x()) / 2, self.__start.y(),
(self.__start.x() + self.__end.x()) / 2, self.__end.y(),
self.__end.x(), self.__end.y())
self.setPath(path)
self.update()
def checkSameConnectorTypeRestriction(self, connector):
""" Check, whether the connector type (top/bottom) is already set in the connection """
if self.__topConnector is not None and connector.isTopConnector():
return False
elif self.__bottomConnector is not None and not connector.isTopConnector():
return False
return True
def getConnectorIfNotFullyConnected(self):
""" Returns the only connected connector, if only one connector is connected """
if self.__topConnector is not None and self.__bottomConnector is None:
return self.__topConnector
elif self.__topConnector is None and self.__bottomConnector is not None:
return self.__bottomConnector
return None
def getHidden(self):
""" Returns whether the connection is hidden """
return self.__hidden
def getNodeEditor(self):
""" Returns the node prototxt_editor object (for the renderer) """
return self.__nodeEditor
def getPhase(self):
""" Returns the internal phase """
if self.getTopConnector() is not None:
return self.getTopConnector().getPhase()
else:
return self.getBottomConnector().getPhase()
def getIsInPlace(self):
""" Returns the internal in-place value """
return self.__isInPlace
def contextMenuEvent(self, event):
""" Creates the context menu """
contextMenu = QMenu()
menuText = "Hide"
if self.__hidden:
menuText = "Show"
# add action to show/hide the connection
toggleHideAction = contextMenu.addAction(menuText)
if not self.__nodeEditor.disable:
removeAction = contextMenu.addAction("Remove")
# show context menu
action = contextMenu.exec_(event.screenPos())
if action is not None:
if action == toggleHideAction:
self.__hidden = not self.__hidden
if self.__hidden:
self.__nodeEditor.tryToAddHiddenConnection(self)
else:
self.__nodeEditor.tryToRemoveHiddenConnection(self)
self.update()
elif not self.__nodeEditor.disable and action == removeAction:
# deselect all Layers
self.__nodeEditor.tryToClearSelection()
# deselect all other connections
for item in self.__nodeEditor.getScene().selectedItems():
item.setSelected(False)
# select this connection
self.setSelected(True)
# remove this connection
self.__nodeEditor.tryToDeleteSelection()
Barista/gui/main_window/node_editor/items/connector_bottom_item.py 0000664 ڊ0011117 00000004713 13265130053 026052 0 ustar klemms o0cvpr from PyQt5.QtWidgets import QMenu
from gui.main_window.node_editor.items.connector_item import ConnectorItem
class ConnectorBottomItem(ConnectorItem):
""" Class to provide bottom connector functionality """
def __init__(self, index, nodeItem, nodeEditor, parent=None):
super(ConnectorBottomItem, self).__init__(index, nodeItem, nodeEditor, parent)
def isTopConnector(self):
""" Returns whether the connector is a top connector (implementation for parent class) """
return False
def isInPlace(self):
""" Returns whether the connector is owned by an in-place working layer
(implementation for parent class) """
return self._nodeItem.getIsInPlace()
def getConnectedNodes(self):
""" Returns a list of node items, connected to this connector (implementation for parent class) """
nodes = list()
# for each connection get the node connected to the top of the connection
for connection in self._connections:
connectionsTopConnector = connection.getTopConnector()
if connectionsTopConnector is not None:
nodes.append(connectionsTopConnector.getNodeItem())
return nodes
def addConnection(self, connection):
""" Adds a connection to the connector and sets the end of the connection to this connectors position
(implementation for parent class) """
self._connections.append(connection)
connection.setEnd(self.scenePos())
def updateConnectionPositions(self):
""" Updates the connected connections, sets the end of all connected connections to this connectors position
(implementation for parent class) """
for connection in self._connections:
connection.setEnd(self.scenePos())
def contextMenuEvent(self, event):
""" Context menu for the bottom connector """
contextMenu = QMenu()
disconnectBottom = contextMenu.addAction("Disconnect")
if self.getConnectionCount() == 0:
disconnectBottom.setEnabled(False)
removeBottom = contextMenu.addAction("Remove")
action = contextMenu.exec_(event.screenPos())
if action is not None:
if action == removeBottom:
self._nodeEditor.tryToRemoveBottomBlob(self._nodeItem.getLayerID(), self._index)
elif action == disconnectBottom:
self._nodeEditor.disconnectBottomBlob(self._nodeItem.getLayerID(), self._index)
Barista/gui/main_window/node_editor/items/connector_item.py 0000664 ڊ0011117 00000010265 13265130053 024465 0 ustar klemms o0cvpr from PyQt5.QtWidgets import QGraphicsItem
from gui.main_window.node_editor.items.renderer.connector_item_renderer import ConnectorItemRenderer
class ConnectorItem(QGraphicsItem):
""" Class to provide basic functionality for top and bottom connectors.
This class should not get instanced. """
def __init__(self, index, nodeItem, nodeEditor, parent=None):
super(ConnectorItem, self).__init__(parent)
# set flag to get position changed to update connected connections
self.setFlag(QGraphicsItem.ItemSendsScenePositionChanges, True)
# draw above the node item
self.setZValue(2.0)
self._nodeEditor = nodeEditor
self._nodeItem = nodeItem
self._index = index
self._blobName = ""
self._connections = list()
# create a renderer to draw the connector
self._renderer = ConnectorItemRenderer(self)
def boundingRect(self):
return self._renderer.boundingRect()
def addConnection(self, connection):
""" Adds a connection to the connector (should get implemented in subclass) """
return
def removeConnection(self, connection):
""" Removes a connection """
self._connections.remove(connection)
def paint(self, painter, option, widget=None):
""" Draws the connector """
self._renderer.paint(painter)
def setIndex(self, index):
""" Sets the index of the connector in the node item """
self._index = index
def isConnected(self):
""" Returns True if there is any connection, False otherwise"""
return len(self._connections) > 0
def isInPlace(self):
""" Returns whether the connector is connected to a in-place working layer
(should get implemented in subclass) """
return
def getNodeEditor(self):
""" Returns the NodeEditor object (for the renderer) """
return self._nodeEditor
def getNodeItem(self):
""" Returns the owning node item """
return self._nodeItem
def setBlobName(self, blobName):
""" Sets the blobs name """
self._blobName = blobName
def getBlobName(self):
""" Returns the blob name """
return self._blobName
def getPhase(self):
""" Returns the phase of the connector (phase of the owning node item) """
return self._nodeItem.getPhase()
def getIndex(self):
""" Returns the index of hte connector in the node item """
return self._index
def isTopConnector(self):
""" Returns whether the connector is a top connector (should get implemented in subclass)"""
return
def updateConnectionPositions(self):
""" Updates the connected connections (should get implemented in subclass) """
return
def getConnectedNodes(self):
""" Returns a list of node items, connected to this connector (should get implemented in subclass) """
return
def getConnections(self):
""" Returns a list of connection items, connected to this connector """
return self._connections
def getConnectionCount(self):
""" Returns the number of connections, connected to this connector """
return len(self._connections)
def hasPhaseConnection(self, phase):
""" Checks whether the connector is connected to a layer in the given phase """
for connection in self._connections:
if connection.getPhase() == "" or connection.getPhase() == phase:
return True
return False
def getConnectedLayers(self):
""" Returns a list of layer IDs of all connected layers """
layerIDs = list()
connectedNodes = self.getConnectedNodes()
for node in connectedNodes:
layerIDs.append(node.getLayerID())
return layerIDs
def itemChange(self, change, value):
""" Update all connections, connected to this connector """
if change == QGraphicsItem.ItemScenePositionHasChanged:
self.updateConnectionPositions()
return super(ConnectorItem, self).itemChange(change, value)
def disableEditing(self, disable):
''' Disable the context menu of the blob. '''
self.setEnabled(not disable) Barista/gui/main_window/node_editor/items/connector_top_item.py 0000664 ڊ0011117 00000005416 13265130053 025351 0 ustar klemms o0cvpr from PyQt5.QtWidgets import QMenu
from gui.main_window.node_editor.items.connector_item import ConnectorItem
class ConnectorTopItem(ConnectorItem):
""" Class to provide top connector functionality """
def __init__(self, index, nodeItem, nodeEditor, parent=None):
super(ConnectorTopItem, self).__init__(index, nodeItem, nodeEditor, parent)
def isTopConnector(self):
""" Returns whether the connector is a top connector (implementation for parent class) """
return True
def isInPlace(self):
""" Returns whether the connector is connected to a in-place working layer
A top connector is in place if any connected bottom connector is in place.
(implementation for parent class) """
for connection in self._connections:
if connection.getIsInPlace():
return True
return False
def getConnectedNodes(self):
""" Returns a list of node items, connected to this connector (implementation for parent class) """
nodes = list()
# for each connection get the node connected to the bottom of the connection
for connection in self._connections:
connectionsBottomConnector = connection.getBottomConnector()
if connectionsBottomConnector is not None:
nodes.append(connectionsBottomConnector.getNodeItem())
return nodes
def addConnection(self, connection):
""" Adds a connection to the connector and sets the start of the connection to this connectors position
(implementation for parent class) """
self._connections.append(connection)
connection.setStart(self.scenePos())
def updateConnectionPositions(self):
""" Updates the connected connections, sets the start of all connected connections to this connectors position
(implementation for parent class) """
for connection in self._connections:
connection.setStart(self.scenePos())
def contextMenuEvent(self, event):
""" Context menu for the top connector """
contextMenu = QMenu()
renameTop = contextMenu.addAction("Change name")
disconnectTop = contextMenu.addAction("Disconnect")
if self.getConnectionCount() == 0:
disconnectTop.setEnabled(False)
removeTop = contextMenu.addAction("Remove")
action = contextMenu.exec_(event.screenPos())
if action is not None:
if action == removeTop:
self._nodeEditor.tryToRemoveTopBlob(self._nodeItem.getLayerID(), self._index)
elif action == renameTop:
self._nodeEditor.tryToRenameTopBlob(self)
elif action == disconnectTop:
self._nodeEditor.disconnectTopBlob(self._nodeItem.getLayerID(), self._index)
Barista/gui/main_window/node_editor/items/node_item.py 0000664 ڊ0011117 00000020362 13265130053 023417 0 ustar klemms o0cvpr from PyQt5.QtCore import QRectF
from PyQt5.QtWidgets import QGraphicsItem
from gui.main_window.node_editor.items.connector_bottom_item import ConnectorBottomItem
from gui.main_window.node_editor.items.connector_top_item import ConnectorTopItem
from gui.network_manager.layer_helper import LayerHelper
from gui.main_window.node_editor.items.renderer.node_item_renderer import NodeItemRenderer
class NodeItem(QGraphicsItem):
def __init__(self, layerID, layerData, nodeEditor, parent=None):
super(NodeItem, self).__init__(parent)
# Tell Qt to send change events
self.setFlag(QGraphicsItem.ItemIsMovable, True)
self.setFlag(QGraphicsItem.ItemIsSelectable, True)
self.setFlag(QGraphicsItem.ItemSendsScenePositionChanges, True)
self.setZValue(1.0)
self.__nodeEditor = nodeEditor
self.__layerID = layerID
self.__layerData = layerData
self.__phase = ""
self.__isInPlace = False
self.__bottomConnectors = list()
self.__topConnectors = list()
# create a renderer to split logic from rendering
self.__renderer = NodeItemRenderer(self)
self.updateLayerData()
def updateLayerData(self):
""" Updates the layers internal variables to represent changes made to the layer and updates the rendering """
self.__phase = LayerHelper.getLayerPhase(self.__layerData)
for topConnector in self.getTopConnectors():
for connection in topConnector.getConnections():
for linkedConnection in connection.getBottomConnector().getConnections():
if self.__phase == "":
if linkedConnection is not connection:
self.__nodeEditor.removeConnection(linkedConnection)
if linkedConnection.getPhase() == self.__phase and linkedConnection is not connection:
self.__nodeEditor.removeConnection(linkedConnection)
self.__updateConnectorNames()
self.__isInPlace = LayerHelper.isLayerInPlace(self.__layerData)
self.__renderer.update()
self.__renderer.updateConnectorPositions(self.__bottomConnectors, True)
self.__renderer.updateConnectorPositions(self.__topConnectors, False)
self.update()
def __updateConnectorNames(self):
""" Sets the internal connector names to correspond to the blob names in the underling data structure """
if "top" in self.__layerData["parameters"]:
for index in range(0, len(self.__topConnectors)):
blobName = self.__layerData["parameters"]["top"][index]
self.__topConnectors[index].setBlobName(blobName)
if "bottom" in self.__layerData["parameters"]:
for index in range(0, len(self.__bottomConnectors)):
blobName = self.__layerData["parameters"]["bottom"][index]
self.__bottomConnectors[index].setBlobName(blobName)
def addTopConnector(self, blobName):
""" Adds a top connector with the given name to the node item """
connector = ConnectorTopItem(len(self.__topConnectors), self, self.__nodeEditor, self)
connector.setBlobName(blobName)
self.__topConnectors.append(connector)
self.updateLayerData()
def addBottomConnector(self, blobName):
""" Adds a bottom connector with the given name to the node item """
connector = ConnectorBottomItem(len(self.__bottomConnectors), self, self.__nodeEditor, self)
connector.setBlobName(blobName)
self.__bottomConnectors.append(connector)
self.updateLayerData()
def removeBottomConnector(self, blobIndex):
""" Removes the bottom connector with the given index """
self.__removeConnector(self.__bottomConnectors, blobIndex)
def removeTopConnector(self, blobIndex):
""" Removes the top connector with the given index """
self.__removeConnector(self.__topConnectors, blobIndex)
def boundingRect(self):
return self.__renderer.boundingRect()
def paint(self, painter, option, widget):
self.__renderer.paint(painter)
def getTopConnectorCount(self):
""" Returns the number of top connectors """
return len(self.__topConnectors)
def getBottomConnectorCount(self):
""" Returns the number of bottom connectors """
return len(self.__bottomConnectors)
def getTopConnectors(self):
""" Returns a list of all top connector items """
return self.__topConnectors
def getBottomConnectors(self):
""" Returns a list of all bottom connector items """
return self.__bottomConnectors
def getNodeEditor(self):
return self.__nodeEditor
def getLayerID(self):
""" Returns the internal layer ID of the layer represented by this node item """
return self.__layerID
def getName(self):
""" Returns the name of the layer, represented by this node item """
return self.__layerData["parameters"]["name"]
def getTypeString(self):
""" Returns the type name as a string """
return self.__layerData["type"].name()
def getType(self):
""" Returns the internal type object """
return self.__layerData["type"]
def getPhase(self):
""" Returns this node items phase """
return self.__phase
def getIsInPlace(self):
""" Returns, whether the layer, represented by this node item, works in-place """
return self.__isInPlace
def getNodesConnectedToTops(self):
""" Returns a list of node items connected to any top connector of this node item """
return self.__getNodesConnectedToConnectors(self.__topConnectors)
def __removeConnector(self, connectors, indexToRemove):
""" Removes the given connector from this node item """
self.scene().removeItem(connectors[indexToRemove])
del connectors[indexToRemove]
self.__updateConnectorIndices(connectors)
self.updateLayerData()
def __updateConnectorIndices(self, connectors):
""" Recalculates the connector indices after deleting a connector """
for i in range(0, len(connectors)):
connectors[i].setIndex(i)
i += 1
def getNodesConnectedToBottoms(self):
""" Returns a list of node items connected to any bottom connector of this node item """
return self.__getNodesConnectedToConnectors(self.__bottomConnectors)
def __getNodesConnectedToConnectors(self, connectors):
""" Returns a list of node items connected to any connector in the given list of connectors """
connected = list()
for connector in connectors:
connectedNodes = connector.getConnectedNodes()
for node in connectedNodes:
if node not in connected:
connected.append(node)
return connected
def itemChange(self, change, value):
""" Check, whether the item is out of the scenes bounds """
if change == QGraphicsItem.ItemPositionChange:
# get the new position
newPos = value
oldRect = self.sceneBoundingRect()
newRect = QRectF(newPos.x(), newPos.y(), oldRect.width(), oldRect.height())
sceneRect = QRectF(self.scene().sceneRect())
# check if the left or right side of the new rect is outside the scenes bounds -> move it back inside
if newRect.left() < sceneRect.left():
newPos.setX(sceneRect.left())
elif newRect.left() + newRect.width() > sceneRect.right():
newPos.setX(sceneRect.right() - newRect.width())
# check if the top or bottom side of the new rect is outside the scenes bounds -> move it back inside
if newRect.bottom() > sceneRect.bottom():
newPos.setY(sceneRect.bottom() - newRect.height())
elif newRect.top() < sceneRect.top():
newPos.setY(sceneRect.top())
return newPos
return super(NodeItem, self).itemChange(change, value)
def disableBlobs(self, disable):
''' Disable the context menu of the tob and bottom blobs. '''
for blob in self.getBottomConnectors():
blob.disableEditing(disable)
for blob in self.getTopConnectors():
blob.disableEditing(disable)
Barista/backend/barista/constraints/__init__.py 0000664 ڊ0011117 00000000546 13265130053 022065 0 ustar klemms o0cvpr """
This package groups some additional constraints required for the Barista program.
Those constraints might not be required to create valid caffe protoxt files, but they are required for the Barista
program due to whatever.
Depending on the subpackages, the included constraints need to be ensured permanently or only at specific times during
runtime.
""" Barista/backend/barista/constraints/common.py 0000664 ڊ0011117 00000000775 13265130053 021622 0 ustar klemms o0cvpr """Common (helper) functions to check constraints."""
def isNumber(input):
"""Check whether input is a valid (float) number."""
try:
float(input)
return True
except ValueError:
return False
def isPositiveNumber(input):
"""Check whether input is a valid (float) number > 0."""
return isNumber(input) and input > 0
def isPositiveInteger(input):
"""Check whether input is a valid integer > 0."""
return isPositiveNumber(input) and float(input).is_integer() Barista/backend/barista/session/__init__.py 0000664 ڊ0011117 00000000000 13265130053 021162 0 ustar klemms o0cvpr Barista/backend/barista/session/client_session.py 0000664 ڊ0011117 00000063251 13265130053 022465 0 ustar klemms o0cvpr from PyQt5.QtCore import QTimer
from backend.barista.session.session import *
from backend.barista.session.session_utils import State
from backend.barista.utils.logger import Log
from backend.parser.parser_dummy import ParserDummy
from backend.networking.net_util import buildTransaction
from backend.networking.protocol import Protocol, SessionProtocol
from gui.main_window.docks.weight_visualization.weights import loadNetParamFromString
class ClientSession(QObject):
# event signals
stateChanged = pyqtSignal(object)
stateDictChanged = pyqtSignal(object, bool)
iterationChanged = pyqtSignal()
snapshotAdded = pyqtSignal(object)
def __init__(self, project, remote, uid, sid):
super(self.__class__, self).__init__()
self.options = {
SessionProtocol.UPDATESTATE: self._updateState,
SessionProtocol.PRINTLOG: self._printLog,
SessionProtocol.UPDATEPARSER: self._updateParser,
SessionProtocol.UPDATEKEYS: self._updateParserKeys,
SessionProtocol.ADDSNAPSHOT: self._addSnapshot,
SessionProtocol.PARSEHANDLE: self._parseHandle
}
self.firstConnect = True
self.remote = remote
self.isConnected = False
self.transaction = None
self.caller_id = None
self.project = project
self.parser = ParserDummy()
# One ID for client, on ID for server
self.sid = sid
self.uid = uid
self.lastStateDict = None
self.state = State.NOTCONNECTED
self.invalidErrorsList = []
# setup timer
# see details in setStateDict()
self.timer = QTimer()
self.timer.timeout.connect(self.transmit)
self.timer.setSingleShot(True)
self.timer.setInterval(10)
self.getStateFirstTime = True
self.lastIter = 0
self.lastMaxIter = 0
# maybe get log
def processMessage(self):
# key = self.transaction.messageOutput[0]["key"]
for subkey in self.options.keys():
if self.transaction.containsAttr(["subkey", subkey]):
self.options[subkey]()
return
self.transaction.stage()
def _assertConnection(self):
if self.isConnected:
return True
return self._connect()
def _connect(self):
if not self.isConnected:
trans = buildTransaction(self.remote[0], self.remote[1])
if trans:
trans.send({"key": Protocol.CONNECTTOSESSION, "uid": self.uid})
ret = trans.asyncRead(attr=("key", Protocol.CONNECTTOSESSION))
if ret:
if ret["status"]:
self.transaction = trans
self.isConnected = True
self.transaction.socketClosed.connect(self._disconnect)
self.transaction.bufferReady.connect(self.processMessage)
self.firstConnect = False
if not self.firstConnect:
self.setState(self.getState(False))
return True
else:
# connected but session not found
self._handleErrors(ret["error"])
trans.close()
return False
else:
# connected but no answer
self._handleErrors(["Communication error on connecting to session."]) # TODO better error msg
trans.close()
return False
else:
# can't connect to host
self._handleErrors(["Host not found"]) # TODO add more info
return False
return True
def _disconnect(self):
if self.isConnected:
self.transaction.socketClosed.disconnect()
self.transaction.bufferReady.disconnect()
self.transaction.close()
self.isConnected = False
self.transaction = None
self.setState(State.NOTCONNECTED)
self.getStateFirstTime = True
self._handleErrors(["Session: Connection to host lost!"])
def _handleErrors(self, errors):
for error in errors:
Log.error(error, self.getCallerId())
def _printLog(self):
msg = self.transaction.asyncRead(attr=("subkey", SessionProtocol.PRINTLOG))
logs = msg["log"]
for log, error in logs:
if not isinstance(log, list):
log = [log]
# if msg["isError"]:
if error:
self._handleErrors(log)
else:
for l in log:
Log.log(l, self.getCallerId())
def _updateIter(self):
msg = self.transaction.asyncRead(attr=("subkey", SessionProtocol.UPDATEITER))
self.lastIter = msg["iteration"]
self.iterationChanged.emit()
def _updateParser(self):
msg = self.transaction.asyncRead(attr=("subkey", SessionProtocol.UPDATEPARSER))
phase = msg["phase"]
row = msg["row"]
self.lastIter = row["NumIters"]
self.iterationChanged.emit()
self.parser.sendParserUpdate(phase, row)
#TODO plotter
def _updateParserKeys(self):
msg = self.transaction.asyncRead(attr=("subkey", SessionProtocol.UPDATEKEYS))
phase = msg["phase"]
key = msg["parserkey"]
self.parser.sendParserRegisterKeys(phase, key)
# Log.log("Found key: " + key, self.getCallerId())
def _addSnapshot(self):
msg = self.transaction.asyncRead(attr=("subkey", SessionProtocol.ADDSNAPSHOT))
snapshot = msg["snapshot"]
self.snapshotAdded.emit(snapshot)
def _parseHandle(self):
msg = self.transaction.asyncRead(attr=("subkey", SessionProtocol.PARSEHANDLE))
event = msg["event"]
message = msg["message"]
groups = msg["groups"]
self.parser.sendParserHandle(event, message, groups)
def getCallerId(self):
""" Return the unique caller id for this session
"""
if self.caller_id is None:
self.caller_id = Log.getCallerId(self._getLogFileName())
return self.caller_id
def _getLogFileName(self):
""" Return the name of the logfile with session id.
"""
log_file = self.project.getProjectName() + '_' + str(self.getSessionId()) + '.log'
return log_file
def getSessionId(self):
return self.sid
def checkFiles(self):
if self._assertConnection():
msg = {"key": Protocol.SESSION, "subkey": SessionProtocol.CHECKFILES}
self.transaction.send(msg)
ret = self.transaction.asyncRead(staging=True, attr=("subkey", SessionProtocol.CHECKFILES))
if ret:
self._handleErrors(ret["error"])
return
self._handleErrors(["Failed to connect to remote session to check files."])
def getSnapshots(self):
""" Return all snapshot files, keyed by iteration number.
"""
if self._assertConnection():
msg = {"key": Protocol.SESSION, "subkey": SessionProtocol.GETSNAPSHOTS}
self.transaction.send(msg)
ret = self.transaction.asyncRead(staging=True, attr=("subkey", SessionProtocol.GETSNAPSHOTS))
if ret:
return ret["snapshots"]
self._handleErrors(["Failed to connect to remote session to acquire Snapshots."])
return {}
def getIteration(self):
# Maybe let this be local + signal
if self._assertConnection():
msg = {"key": Protocol.SESSION, "subkey": SessionProtocol.GETITERATION}
self.transaction.send(msg)
ret = self.transaction.asyncRead(staging=True, attr=("subkey", SessionProtocol.GETITERATION))
if ret:
self.lastIter = ret["iteration"]
return self.lastIter
self._handleErrors(["Failed to connect to remote session to acquire current iteration."])
return 0
def getMaxIteration(self):
if self.lastMaxIter > 0 \
and self.getState() in [State.RUNNING, State.FAILED, State.FINISHED, State.NOTCONNECTED]:
return self.lastMaxIter
elif self._assertConnection():
msg = {"key": Protocol.SESSION, "subkey": SessionProtocol.GETMAXITERATION}
self.transaction.send(msg)
ret = self.transaction.asyncRead(staging=True, attr=("subkey", SessionProtocol.GETMAXITERATION))
if ret:
self.lastMaxIter = ret["iteration"]
return self.lastMaxIter
self._handleErrors(["Failed to connect to remote session to acquire max iteration."])
return 1
def getPretrainedWeights(self):
if self._assertConnection():
msg = {"key": Protocol.SESSION, "subkey": SessionProtocol.GETPRETRAINED}
self.transaction.send(msg)
ret = self.transaction.asyncRead(staging=True, attr=("subkey", SessionProtocol.GETPRETRAINED))
if ret:
return ret.get("pretrained", None)
self._handleErrors(["Failed to connect to remote session to acquire pre-trained weights."])
return None
def setMaxIteration(self, maxIteration):
if maxIteration <= 0:
return
if self._assertConnection():
msg = {"key": Protocol.SESSION, "subkey": SessionProtocol.SETMAXITERATION, "iteration": maxIteration}
self.transaction.send(msg)
self.lastMaxIter = maxIteration
else:
self._handleErrors(["Failed to connect to remote session to update max iteration."])
self.setState(State.NOTCONNECTED, True)
def delete(self):
if self._assertConnection():
msg = {"key": Protocol.DELETESESSION, "uid": self.uid}
self.transaction.send(msg)
ret = self.transaction.asyncRead(staging=True, attr=("key", Protocol.DELETESESSION))
if ret:
if ret["status"]:
self.project.deleteSession.emit(self.getSessionId())
else:
self._handleErrors(["Failed to connect to remote session to acquire current state."])
self.setState(State.NOTCONNECTED, True)
def close(self):
if self._assertConnection():
msg = {"key": Protocol.DISCONNECTSESSION, "uid": self.uid}
self.transaction.send(msg)
def reset(self):
if self._assertConnection():
msg = {"key": Protocol.SESSION, "subkey": SessionProtocol.RESET}
self.transaction.send(msg)
ret = self.transaction.asyncRead(staging=True, attr=("subkey", SessionProtocol.RESET))
if ret:
if ret["status"]:
self.iterationChanged.emit()
self.project.resetSession.emit(self.getSessionId())
return True
else:
self._handleErrors(ret["error"])
self._handleErrors(["Could not reset session in state " + str(self.getState())])
return False
def getState(self, local=True):
if local and not self.getStateFirstTime:
return self.state
self.getStateFirstTime = False
if self._assertConnection():
msg = {"key": Protocol.SESSION, "subkey": SessionProtocol.GETSTATE}
self.transaction.send(msg)
ret = self.transaction.asyncRead(staging=True, attr=("subkey", SessionProtocol.GETSTATE))
if ret:
if ret["status"]:
self.setState(ret["state"], True)
return ret["state"]
else:
self._handleErrors(ret["error"])
self.setState(State.UNDEFINED, True)
return State.UNDEFINED
self._handleErrors(["Failed to connect to remote session to acquire current state."])
self.setState(State.NOTCONNECTED, True)
return State.NOTCONNECTED
def setState(self, state, silent=False):
self.state = state
if not silent:
self.stateChanged.emit(self.state)
def setErrorList(self, errorList):
self.invalidErrorsList = errorList
def _updateState(self):
msg = self.transaction.asyncRead(attr=("subkey", SessionProtocol.UPDATESTATE))
self.setState(msg["state"])
def getErrorList(self):
return self.invalidErrorsList
def save(self):
Log.log("Saving current Session status to disk.", self.getCallerId())
if self._assertConnection():
msg = {"key": Protocol.SESSION, "subkey": SessionProtocol.SAVE}
self.transaction.send(msg)
ret = self.transaction.asyncRead(attr=("subkey", SessionProtocol.SAVE))
if ret:
if ret["status"]:
return True
else:
self._handleErrors(ret["error"])
self._handleErrors(["Could not save session."])
return False
def setStateDict(self, stateDict):
self.lastStateDict = stateDict
# to prevent subsequent writes, wait until some time passed before sending the statedict
self.timer.stop()
self.timer.start()
def transmit(self):
if self._assertConnection():
if not self._validateLayerOrder(self.lastStateDict):
self._handleErrors(["Warning: layerorder does not match layers"])
return
msg = {"key": Protocol.SESSION, "subkey": SessionProtocol.SETSTATEDICT}
# remove types for transfer
transferDict = copy.deepcopy(self.lastStateDict)
layers = transferDict["network"]["layers"]
for id in layers:
del layers[id]["type"]
msg["statedict"] = transferDict
self.transaction.send(msg)
ret = self.transaction.asyncRead(attr=("subkey", SessionProtocol.SETSTATEDICT))
if ret:
if not ret["status"]:
self._handleErrors(ret["error"])
else:
self._handleErrors(["SetStateDict: Failed to send StateDict"]) # TODO improve warnings
self.stateDictChanged.emit(self, False)
def _validateLayerOrder(self, dict):
"""There is a fundamental problem in how the setStateDict function is designed.
In theory there should by only one write one write access.
But in reality layerorder and the layer itself are writen in subsequent operations.
this leads to a statedict with a layer in the layerorder, but no corresponding layer in the net.
thus the save function produces a key error since they iterate over the layerorder.
simply waiting until the layer is in layerorder and layer solves this problem.
this function verifies this for all IDs.
"""
for id in dict["network"]["layerOrder"]:
if id not in dict["network"]["layers"]:
return False
return True
@property
def state_dictionary(self):
if self.lastStateDict is None:
if self._assertConnection():
msg = {"key": Protocol.SESSION, "subkey": SessionProtocol.GETSTATEDICT}
self.transaction.send(msg)
ret = self.transaction.asyncRead(staging=True, attr=("subkey", SessionProtocol.GETSTATEDICT))
if ret:
if ret["status"]:
self.lastStateDict = ret["statedict"]
try:
layers = self.lastStateDict["network"]["layers"]
for id in layers:
if "parameters" in layers[id]:
if "type" in layers[id]["parameters"]:
typename = layers[id]["parameters"]["type"]
layers[id]["type"] = info.CaffeMetaInformation().availableLayerTypes()[typename]
except KeyError:
pass
return self.lastStateDict
else:
self._handleErrors(ret["error"])
if self.lastStateDict is None:
return {u'position': {}, u'selection': [], u'hidden_connections': {},
u'network': {u'layers': {}, u'name': u'default', u'layerOrder': []},
u'solver': {u'net': u'net-internal.prototxt'}}
return self.lastStateDict
@state_dictionary.setter
def state_dictionary(self, value):
self.setStateDict(value)
def hasStateDict(self):
return True
def checkTraining(self):
if self._assertConnection():
msg = {"key": Protocol.SESSION, "subkey": SessionProtocol.CHECKTRAINING}
self.transaction.send(msg)
ret = self.transaction.asyncRead(staging=True, attr=("subkey", SessionProtocol.CHECKTRAINING))
if ret:
if ret["status"]:
return ret["check"]
else:
self._handleErrors(msg["error"])
return [(msg["error"], "")]
else:
self._handleErrors(["CheckTraining: Did not receive a reply from host."])
return [("CheckTraining: Did not receive a reply from host.", "No reply from host")]
else:
self._handleErrors(["CheckTraining: Failed to connect to host to check session validity!"])
return [("CheckTraining: Failed to connect to host to check session validity!", "Failed to connect")]
def start(self, solverstate=None, caffemodel=None):
if self._assertConnection():
msg = {"key": Protocol.SESSION, "subkey": SessionProtocol.START}
if solverstate is not None:
msg["solverstate"] = solverstate
if caffemodel is not None:
msg["caffemodel"] = caffemodel
self.transaction.send(msg)
ret = self.transaction.asyncRead(staging=True, attr=("subkey", SessionProtocol.START))
if ret:
if ret["status"]:
return True
else:
self._handleErrors(ret["error"])
self._handleErrors(["Could not start session in state " + str(self.getState())])
return False
def pause(self):
if self._assertConnection():
msg = {"key": Protocol.SESSION, "subkey": SessionProtocol.PAUSE}
self.transaction.send(msg)
ret = self.transaction.asyncRead(staging=True, attr=("subkey", SessionProtocol.PAUSE))
if ret:
if ret["status"]:
self.iterationChanged.emit()
return True
else:
self._handleErrors(ret["error"])
self._handleErrors(["Could not pause session in state " + str(self.getState())])
return False
def snapshot(self):
if self._assertConnection():
msg = {"key": Protocol.SESSION, "subkey": SessionProtocol.TAKESNAPSHOT}
self.transaction.send(msg)
ret = self.transaction.asyncRead(staging=True, attr=("subkey", SessionProtocol.TAKESNAPSHOT))
if ret:
if ret["status"]:
return True
else:
self._handleErrors(ret["error"])
self._handleErrors(["Could not take snapshot of session in state " + str(self.getState())])
return False
def proceed(self, snapshot=None):
# TODO snapshot
if self._assertConnection():
msg = {"key": Protocol.SESSION, "subkey": SessionProtocol.PROCEED}
if snapshot is not None:
msg["snapshot"] = snapshot
print "proceed with snapshot", msg
self.transaction.send(msg)
ret = self.transaction.asyncRead(staging=True, attr=("subkey", SessionProtocol.PROCEED))
if ret:
if ret["status"]:
return True
else:
self._handleErrors(ret["error"])
self._handleErrors(["Could not proceed session in state " + str(self.getState())])
return False
def getParser(self):
return self.parser
def getLogFileName(self, basename=False):
if basename:
return self.project.getProjectName() + "_" + str(self.sid) + ".log"
else:
print "There is no 'getLogFileName' for clientsessions!"
return None
def getLogId(self):
return self.uid
def isRemote(self):
return True
def fetchParserData(self):
if self._assertConnection() and self.transaction is not None:
msg = {"key": Protocol.SESSION, "subkey": SessionProtocol.FETCHPARSERDATA}
self.transaction.send(msg)
ret = self.transaction.asyncRead(staging=True, attr=("subkey", SessionProtocol.FETCHPARSERDATA))
if ret:
if ret["status"]:
parserRows = ret["ParserRows"]
parserKeys = ret["ParserKeys"]
parserHandle = ret["ParserHandle"]
parserLog = ret["ParserLogs"]
for phase, key in parserKeys:
self.parser.sendParserRegisterKeys(phase, key)
for phase, row in parserRows:
self.parser.sendParserUpdate(phase, row)
for event, line, groups in parserHandle:
self.parser.sendParserHandle(event, line, groups)
for log, error in parserLog:
if error:
Log.error(log, self.getCallerId())
else:
Log.log(log, self.getCallerId())
return
else:
self._handleErrors(ret["error"])
self._handleErrors(["Failed to fetch parser data from host."])
def readInternalNetFile(self):
""" Returns the contents of the internal net prototxt file.
Wrapper around loadInternalNetFile, to provide a coherent interface
together with the Session class.
"""
return self.loadInternalNetFile()
def readDeployedNetAsString(self):
""" Returns the contents of the deployable net prototxt file.
Wrapper around loadDeployedNetFile, to provide a coherent interface
together with the Session class.
"""
return self.loadDeployedNetAsString()
def readCaffemodelFile(self, snapshot):
""" Returns the contents of the .caffemodel file that belongs to snapshot.
snapshot: string
Filename without path of the snapshot file.
"""
return self.loadCaffemodel(snapshot)
def loadInternalNetFile(self):
if self._assertConnection():
msg = {"key": Protocol.SESSION, "subkey": SessionProtocol.LOADINTERNALNET}
self.transaction.send(msg)
ret = self.transaction.asyncRead(staging=True, attr=("subkey", SessionProtocol.LOADINTERNALNET))
if ret:
if ret["status"]:
return ret["NetInternal"]
else:
self._handleErrors(ret["error"])
self._handleErrors(["Failed to load InternalNet Prototxt"])
return ""
def loadDeployedNetAsString(self):
if self._assertConnection():
msg = {"key": Protocol.SESSION, "subkey": SessionProtocol.LOADDEPLOYEDNET}
self.transaction.send(msg)
ret = self.transaction.asyncRead(staging=True, attr=("subkey", SessionProtocol.LOADDEPLOYEDNET))
if ret:
if ret["status"]:
return ret["Net"]
else:
self._handleErrors(ret["error"])
self._handleErrors(["Failed to load Deployed Net"])
return ""
def loadNetParameter(self, snapshot):
if self._assertConnection():
msg = {"key": Protocol.SESSION, "subkey": SessionProtocol.LOADNETPARAMETER}
msg["snapshot"] = snapshot
self.transaction.send(msg)
ret = self.transaction.asyncRead(staging=True, attr=("subkey", SessionProtocol.LOADNETPARAMETER))
if ret:
if ret["status"]:
net = ret["NetParams"]
net = loadNetParamFromString(net)
if net is not None:
return net
else:
self._handleErrors(["Failed to load " + str(snapshot),
"HDF5 snapshot format is not supported for weight "
"visualization. This can be changed by setting the "
"snapshot_format in the solver properties."])
else:
self._handleErrors(ret["error"])
self._handleErrors(["Failed to load NetParameter for snapshot '" + snapshot + "'"])
return None
def getCaffemodelContents(self, snapshot):
""" Wrapper around the loadCaffemodel function to provide a coherent interface
between local and remote sessions. """
return self.loadCaffemodel(snapshot)
def loadCaffemodel(self, snapshot):
""" Loads the contents of the caffemodel file belonging to the given snapshot
from the server. """
if self._assertConnection():
msg = {"key": Protocol.SESSION, "subkey": SessionProtocol.LOADCAFFEMODEL}
msg["snapshot"] = snapshot
self.transaction.send(msg)
ret = self.transaction.asyncRead(staging=True, attr=("subkey", SessionProtocol.LOADCAFFEMODEL))
if ret:
if ret["status"]:
return ret["caffemodel"]
else:
self._handleErrors(ret["error"])
self._handleErrors(["Failed to load caffemodel contents for snapshot '" + snapshot + "'"])
return None
Barista/backend/barista/session/server_session.py 0000664 ڊ0011117 00000125230 13265130053 022511 0 ustar klemms o0cvpr import copy
import json
import os
import re
import signal
import sys
import time
import uuid
import shutil
from collections import OrderedDict
from subprocess import Popen, PIPE, STDOUT
from PyQt5.Qt import QObject
from PyQt5.QtCore import Qt, pyqtSignal
import backend.caffe.dict_helper as helper
import backend.caffe.proto_info as info
import backend.caffe.saver as saver
from backend.barista.constraints.session_run.training import checkMinimumTrainingRequirements
from backend.barista.session.session import State
from backend.barista.session.session_common import SessionCommon
from backend.barista.session.session_pool import SessionPool
from backend.barista.session.session_utils import Paths, Events
from backend.networking.protocol import Protocol, SessionProtocol
from backend.parser.concatenator import Concatenator
from backend.parser.parser import Parser
from backend.parser.parser_listener import ParserListener
import backend.barista.caffe_versions as caffeVersions
from backend.barista.deployed_net import DeployedNet
from backend.caffe.proto_info import UnknownLayerTypeException
from PyQt5.QtCore import QTimer
from threading import Lock
class ServerSession(QObject, ParserListener, SessionCommon):
logsig = pyqtSignal(str, bool) # This is the way to do threadsafe logging while the session is running
parssig = pyqtSignal(str, OrderedDict)
keysig = pyqtSignal(str, str)
statesig = pyqtSignal(int)
snapshotsig = pyqtSignal(str)
handlesig = pyqtSignal(str, str, list)
# a runnning session is moved to another thread (which is not a QThread and has no main-loop)
# using the network feature causes warnings unless every operation is connected with a signal to the
# original network method with Qt.Autoconnect (theoretically Qt.QueuedConnection should work as well)
def __init__(self, manager, directory, parse_old=False, pid=""):
super(self.__class__, self).__init__()
self.options = {SessionProtocol.CHECKFILES: self._msgCheckFiles,
SessionProtocol.CHECKTRAINING: self._msgCheckTraining,
SessionProtocol.GETSNAPSHOTS: self._msgGetSnapshots,
SessionProtocol.GETITERATION: self._msgGetIteration,
SessionProtocol.GETMAXITERATION: self._msgGetMaxIteration,
SessionProtocol.SETMAXITERATION: self._msgSetMaxIteration,
SessionProtocol.GETSTATE: self._msgGetState,
SessionProtocol.GETPRETRAINED: self._msgGetPretrainedWeights,
SessionProtocol.SETSTATEDICT: self._msgSetStateDict,
SessionProtocol.GETSTATEDICT: self._msgGetStateDict,
SessionProtocol.SAVE: self._msgSave,
SessionProtocol.START: self._start,
SessionProtocol.PAUSE: self._pause,
SessionProtocol.PROCEED: self._proceed,
SessionProtocol.TAKESNAPSHOT: self._takeSnapshot,
SessionProtocol.FETCHPARSERDATA: self._msgFetchParserData,
SessionProtocol.LOADINTERNALNET: self._msgLoadInternalNet,
SessionProtocol.LOADDEPLOYEDNET: self._msgLoadDeployedNet,
SessionProtocol.LOADNETPARAMETER: self._msgLoadNetParameter,
SessionProtocol.LOADCAFFEMODEL: self._msgLoadCaffemodel,
SessionProtocol.RESET: self._reset,
SessionProtocol.DELETE: self.delete}
self.logsig.connect(self.Log, Qt.AutoConnection)
self.parssig.connect(self.addParserRow, Qt.AutoConnection)
self.keysig.connect(self.addParserKey, Qt.AutoConnection)
self.statesig.connect(self.setState, Qt.AutoConnection)
self.snapshotsig.connect(self.addSnapshot, Qt.AutoConnection)
self.handlesig.connect(self.addHandle, Qt.AutoConnection)
self.transaction = None
self.isConnected = False
self.manager = manager
self.directory = directory
self.pid = None
self.sid = None
self.uid = str(uuid.uuid4())
self.snapshot_dir = None
self.snapshot_prefix = None
self.iteration = 0
self.max_iter = 1
self.state_dictionary = {}
self.state = State.WAITING
self.invalidErrorsList = []
self.last_solverstate = None
self.pretrainedWeights = None
self.logs = os.path.join(directory, 'logs')
# run stuff
self.rid = 0
self.tee = None
self.proc = None
self.parser = None
self.lock = Lock()
self.parse_old = False
self.parser_initialized = False
self.parserRows = []
self.parserKeys = []
self.parserHandle = []
self.parserLogs = []
self.timer = QTimer()
self.timer.timeout.connect(self.transmitLog)
self.timer.setSingleShot(True)
self.timer.setInterval(10)
self.logBuffer = []
self.logLock = Lock()
if not parse_old:
self.pid = pid
else:
self.rebuild()
self._parseSessionAndRunID()
self.parse_old = True
def __lt__(self, other):
if type(other) is not ServerSession:
return True
return other.sid - self.sid < 0
def __gt__(self, other):
if type(other) is not ServerSession:
return False
return other.sid - self.sid > 0
def __eq__(self, other):
if type(other) is not ServerSession:
return False
return other.sid - self.sid == 0
def __le__(self, other):
if type(other) is not ServerSession:
return True
return other.sid - self.sid <= 0
def __ge__(self, other):
if type(other) is not ServerSession:
return False
return other.sid - self.sid >= 0
def __ne__(self, other):
if type(other) is not ServerSession:
return True
return other.sid - self.sid != 0
def rebuild(self):
filename = os.path.join(self.directory, Paths.FILE_NAME_SESSION_JSON)
if os.path.isfile(filename):
with open(filename, 'r') as f:
try:
res = json.load(f)
self._parseSetting(res)
except Exception:
sys.stderr.write("ERROR on reading json data from: " + filename + "\n")
def connect(self, transaction):
if not self.isConnected:
self.isConnected = True
self.transaction = transaction
self.transaction.bufferReady.connect(self.processMessage)
self.transaction.socketClosed.connect(self.disconnect)
return True
return False
def disconnect(self):
self.isConnected = False
if self.transaction is not None:
self.transaction.bufferReady.disconnect()
self.transaction.socketClosed.disconnect()
self.transaction = None
def processMessage(self):
# key = self.transaction.messageOutput[0]["key"]
key, subkey = self.transaction.getAttrOfFirst(["key", "subkey"])
if key is Protocol.SESSION:
# if "subkey" in self.transaction.messageOutput[0]:
# subkey = self.transaction.messageOutput[0]["subkey"]
if subkey is not None:
if subkey in self.options:
self.options[subkey]()
else:
sys.stderr.write("Wrong subkey for session: " + str(subkey) + "\n")
msg = self.transaction.asyncRead()
msg["status"] = False
msg["error"] = ["Wrong subkey for session: " + str(subkey)]
self.transaction.send(msg)
return
else:
sys.stderr.write("Missing subkey for session\n")
msg = self.transaction.asyncRead()
msg["status"] = False
msg["error"] = ["Missing subkey for session"]
self.transaction.send(msg)
return
def setErrorList(self, errorList):
self.invalidErrorsList = errorList
def setState(self, state):
self.state = state
if (self.isConnected):
msg = {"key": Protocol.SESSION, "subkey": SessionProtocol.UPDATESTATE}
msg["state"] = state
self.transaction.send(msg)
def Log(self, log, error=False):
self.logLock.acquire()
self.logBuffer.append((log, error))
self.logLock.release()
self.parserLogs.append((log, error))
self.timer.stop()
self.timer.start()
# if self.isConnected:
# msg = {"key": Protocol.SESSION, "subkey": SessionProtocol.PRINTLOG}
# msg["log"] = log
# msg["isError"] = error
# self.transaction.send(msg)
def transmitLog(self):
self.logLock.acquire()
log = self.logBuffer
self.logBuffer = []
self.logLock.release()
if self.isConnected:
msg = {"key": Protocol.SESSION, "subkey": SessionProtocol.PRINTLOG}
msg["log"] = log
self.transaction.send(msg)
def addParserRow(self, phase, row):
self.parserRows.append((phase, row))
if self.isConnected:
msg = {"key": Protocol.SESSION, "subkey": SessionProtocol.UPDATEPARSER, "phase": phase, "row": row}
self.transaction.send(msg)
def addParserKey(self, phase, key):
self.parserKeys.append((phase, key))
if self.isConnected:
msg = {"key": Protocol.SESSION, "subkey": SessionProtocol.UPDATEKEYS, "phase": phase, "parserkey": key}
self.transaction.send(msg)
def addSnapshot(self, snapshot):
if self.isConnected:
msg = {"key": Protocol.SESSION, "subkey": SessionProtocol.ADDSNAPSHOT, "snapshot": snapshot}
self.transaction.send(msg)
def addHandle(self, event, message, groups):
self.parserHandle.append([event, message, groups])
if self.isConnected:
msg = {"key": Protocol.SESSION, "subkey": SessionProtocol.PARSEHANDLE,
"event": event, "message": message, "groups": groups}
self.transaction.send(msg)
def getDirectory(self):
return self.directory
def getErrorList(self):
return self.invalidErrorsList
def setInitialSid(self, sid):
self.sid = sid
def _msgCheckFiles(self):
""" Check for the existence of the session directories and files.
"""
msg = self.transaction.asyncRead()
errors = []
if os.path.exists(self.getDirectory()) is False:
errors.append('Session directory does not exists: ' + self.directory)
if os.path.exists(self.logs) is False:
errors.append('Log directory does not exists: ' + self.logs)
if os.path.exists(self.snapshot_dir) is False:
errors.append('Snapshot directory does not exists: ' + self.snapshot_dir)
if os.path.exists(caffeVersions.getDefaultVersion().getBinaryPath()) is False:
errors.append('CAFFE_BINARY does not exists: ' + caffeVersions.getDefaultVersion().getBinaryPath())
msg["error"] = errors
self.transaction.send(msg)
def _msgCheckTraining(self):
msg = self.transaction.asyncRead()
req = checkMinimumTrainingRequirements(self)
self.setState(self._getState())
msg["status"] = True
msg["check"] = req
self.transaction.send(msg)
def delete(self):
self.pause() # make sure the session is not running
try:
shutil.rmtree(self.getDirectory())
except OSError as e:
sys.stderr.write(str(e)+'\n') # python docs say, that rmtree should raise an OSError code 66 if dir is not empty. However, this does not seem to happen.
self.disconnect()
return
def _msgGetSnapshots(self):
""" Return all snapshot files, keyed by iteration number.
"""
msg = self.transaction.asyncRead()
msg["snapshots"] = self._getSnapshots()
self.transaction.send(msg)
def _getSnapshots(self):
regex_snapshot = re.compile('iter_([\d]+)\.solverstate')
snaps = {}
if os.path.exists(self.getSnapshotDirectory()):
for entry in os.listdir(self.getSnapshotDirectory()):
snap_match = regex_snapshot.search(entry)
if snap_match:
try:
iter_num = int(snap_match.group(1))
snaps[iter_num] = entry
except Exception:
pass
return snaps
def _msgGetIteration(self):
msg = self.transaction.asyncRead()
msg["iteration"] = self.getIteration()
self.transaction.send(msg)
def _msgGetMaxIteration(self):
msg = self.transaction.asyncRead()
msg["iteration"] = self.getMaxIteration()
self.transaction.send(msg)
def _msgSetMaxIteration(self):
msg = self.transaction.asyncRead()
self.max_iter = msg["iteration"]
def _msgGetState(self):
msg = self.transaction.asyncRead()
msg["state"] = self._getState()
msg["status"] = True
self.transaction.send(msg)
def _msgGetPretrainedWeights(self):
msg = self.transaction.asyncRead()
msg["pretrained"] = self.getPretrainedWeights()
self.transaction.send(msg)
def getLastModel(self):
return self.last_caffemodel
def getIteration(self):
return self.iteration
def getMaxIteration(self):
return self.max_iter
def getPretrainedWeights(self):
return self.pretrainedWeights
def setPretrainedWeights(self, weights):
self.pretrainedWeights = weights
def _getState(self):
""" Return the state of the session.
"""
if self.state == State.FAILED:
return self.state
if self.proc is not None:
if self.iteration == self.max_iter:
self.state = State.FINISHED
else:
self.state = State.RUNNING
else:
if len(checkMinimumTrainingRequirements(self)) > 0:
self.state = State.INVALID
elif self.iteration == self.max_iter:
self.state = State.FINISHED
elif len(self._getSnapshots()) > 0:
self.state = State.PAUSED
else:
self.state = State.WAITING
return self.state
def _msgSetStateDict(self):
msg = self.transaction.asyncRead()
try:
self.setStateDict(msg["statedict"])
msg["status"] = True
except UnknownLayerTypeException as e:
msg["status"] = False
msg["error"] = [e._msg]
del msg["statedict"]
self.transaction.send(msg)
def setStateDict(self, statedict):
self.state_dictionary = statedict
self._parseSetting(self.state_dictionary)
# restore lost types
if hasattr(self.state_dictionary, '__getitem__'):
if "network" in self.state_dictionary:
if "layers" in self.state_dictionary["network"]:
layers = self.state_dictionary["network"]["layers"]
for id in layers:
if "parameters" in layers[id]:
if "type" in layers[id]["parameters"]:
typename = layers[id]["parameters"]["type"]
layers[id]["type"] = info.CaffeMetaInformation().getLayerType(typename)
def _msgGetStateDict(self):
msg = self.transaction.asyncRead()
transferDict = copy.deepcopy(self.state_dictionary)
try:
layers = transferDict["network"]["layers"]
for id in layers:
del layers[id]["type"]
msg["statedict"] = transferDict
msg["status"] = True
except KeyError as e:
msg["statedict"] = transferDict
msg["status"] = True
self.transaction.send(msg)
def save(self, includeProtoTxt = False, errors = []):
"""Saves the current session to prototxt files and session_settings json file."""
res = self.__ensureDirectory()
if len(res) > 0:
return res
availableTypes = info.CaffeMetaInformation().availableLayerTypes()
unknownLayerTypes = []
for layerID in self.state_dictionary["network"]["layers"]:
type = self.state_dictionary["network"]["layers"][layerID]["parameters"]["type"]
if not type in availableTypes and not type in unknownLayerTypes:
unknownLayerTypes.append(type)
if len(unknownLayerTypes) > 0:
errors.extend(unknownLayerTypes)
return False
else:
toSave = {"SessionState": self.state, "Iteration": self.iteration, "MaxIter": self.max_iter}
toSave["UID"] = self.uid
toSave["SID"] = self.sid
toSave["ProjectID"] = self.pid
if self.last_solverstate:
toSave["LastSnapshot"] = self.last_solverstate
if self.pretrainedWeights:
toSave["PretrainedWeights"] = self.pretrainedWeights
if self.state_dictionary:
serializedDict = copy.deepcopy(self.state_dictionary)
if "network" in serializedDict:
if includeProtoTxt:
netDict = copy.deepcopy(self.state_dictionary["network"])
net = saver.saveNet(netdict=netDict)
with open(os.path.join(self.directory, Paths.FILE_NAME_NET_ORIGINAL), 'w') as f:
f.write(net)
if "layers" in serializedDict["network"]:
layers = serializedDict["network"]["layers"]
for id in layers:
del layers[id]["type"]
toSave["NetworkState"] = serializedDict
filename = os.path.join(self.directory, Paths.FILE_NAME_SESSION_JSON)
# clear the file. sometimes json.dump does not exitclean and causes valuerrors on load
open(filename, 'w').close()
with open(filename, "w") as f:
json.dump(toSave, f, sort_keys=True, indent=4)
return True
def prepairInternalPrototxt(self):
error = []
serializedDict = copy.deepcopy(self.state_dictionary)
if "solver" in self.state_dictionary:
solverDict = self.state_dictionary["solver"]
if self.manager.parent.trainOnHW == 0:
solverDict["solver_mode"] = 'CPU'
else:
solverDict["solver_mode"] = 'GPU'
solver = saver.saveSolver(solverdict=solverDict)
with open(os.path.join(self.directory, Paths.FILE_NAME_SOLVER), 'w') as f:
f.write(solver)
else:
error.append("There is no solver")
if "network" in serializedDict:
net, error = self._modifyNetDictionaryToInternalVersion(serializedDict["network"])
net = saver.saveNet(net)
with open(os.path.join(self.directory, Paths.FILE_NAME_NET_INTERNAL), 'w') as f:
f.write(net)
else:
error.append("There is no network")
return error
def _modifyNetDictionaryToInternalVersion(self, net):
# define all layer parameters that can contain (relative) paths
# (use a dot to separate nested parameters)
layerParamsContainingPaths = [
"data_param.source",
"hdf5_data_param.source",
"image_data_param.source",
"window_data_param.source",
"data_param.mean_file",
"hdf5_output_param.file_name",
"image_data_param.mean_file",
"window_data_param.mean_file",
"transform_param.mean_file"]
error = []
# evaluate layer after layer
h = helper.DictHelper(net)
for layerId, layer in net.get("layers", {}).iteritems():
# evaluate parameter after parameter
for paramKey in layerParamsContainingPaths:
# if the current layer does have the current parameter..
if h.layerParameterIsSet(layerId, paramKey):
paramValue = h.layerParameter(layerId, paramKey)
# ..and its value is a relative path: modify it
if paramValue is not None and not os.path.isabs(paramValue):
if paramKey != "hdf5_data_param.source":
newPath = os.path.join(os.pardir, paramValue)
else:
newFilename = str(layerId) + ".txt"
newPath = os.path.join(os.curdir, newFilename)
parentPath = self.manager.parent.sessionPath
oldPath = os.path.join(parentPath, paramValue)
# this builds the new hdf5 files
if os.path.exists(oldPath):
with open(os.path.join(self.directory, newPath), 'w') as f:
lines = [line.rstrip('\n') for line in open(oldPath)]
for line in lines:
if line is not "":
if line[:1] == '.':
line = os.path.join(os.pardir, line)
f.write("\n" + line)
else:
self.Log("Failed to copy hdf5txt file. File does not exists: " + oldPath, True)
error.append("Failed to copy hdf5txt file. File does not exists: " + oldPath)
h.setLayerParameter(layerId, paramKey, newPath)
return net, error
def _parseSetting(self, settings):
if not hasattr(settings, '__getitem__'):
return
else:
if "UID" in settings:
self.uid = settings["UID"]
if "SID" in settings:
self.sid = settings["SID"]
if "ProjectID" in settings:
self.pid = settings["ProjectID"]
if "SessionState" in settings:
self.state = settings["SessionState"]
if self.state == State.RUNNING:
self.state = State.PAUSED
self.previousState = self.state
if "Iteration" in settings:
self.iteration = settings["Iteration"]
if "solver" in settings:
if "max_iter" in settings["solver"]:
self.max_iter = settings["solver"]["max_iter"]
elif "MaxIter" in settings:
self.max_iter = settings["MaxIter"]
if "LastSnapshot" in settings:
self.last_solverstate = settings["LastSnapshot"]
if "PretrainedWeights" in settings:
self.pretrainedWeights = settings["PretrainedWeights"]
if "NetworkState" in settings:
self.state_dictionary = settings["NetworkState"]
layers = self.state_dictionary["network"]["layers"]
for id in layers:
if "parameters" in layers[id]:
if "type" in layers[id]["parameters"]:
typename = layers[id]["parameters"]["type"]
layers[id]["type"] = info.CaffeMetaInformation().getLayerType(typename)
solver = self.state_dictionary["solver"]
if solver:
if "snapshot_prefix" in solver:
self.snapshot_prefix = solver["snapshot_prefix"]
def _parseSessionAndRunID(self):
if self.sid is None:
regex_sid = re.compile('[\d]{8}_[\d]{6}_([\d]+)')
sid_match = regex_sid.search(self.directory)
session_id = None
if sid_match:
try:
self.sid = int(sid_match.group(1))
except:
pass
if self.sid is None:
self.sid = 0
regex_rid = re.compile('([\d]+)\.([\d]+)\.log')
run_id = 0
if os.path.exists(self.logs):
for entry in os.listdir(self.logs):
rid_match = regex_rid.search(entry)
if rid_match:
try:
_run_id = int(rid_match.group(2))
if _run_id > run_id:
run_id = _run_id
except:
pass
self.rid = run_id
def getSnapshotDirectory(self):
""" Return the snapshot directory.
"""
return self.getDirectory()
# if self.snapshot_dir:
# return self.snapshot_dir
# snapshot_prefix = self._getSnapshotPrefix()
# sdir = os.path.dirname(snapshot_prefix)
# self.snapshot_dir = os.path.join(self.directory, sdir)
# return self.snapshot_dir
def _getSnapshotPrefix(self):
""" Return the snapshot prefix which is used for snapshots of this
session.
"""
if not self.snapshot_prefix:
pass
# self.__parseSnapshotPrefix() # TODO
if not self.snapshot_prefix:
self.snapshot_prefix = ''
return self.snapshot_prefix
def _start(self):
msg = self.transaction.asyncRead()
if self.manager.isTraining():
msg["error"] = ["Could not start Session. There is already one running Session on this Server."]
msg["status"] = False
self.transaction.send(msg)
return
solverstate = None
if "solverstate" in msg:
solverstate = msg["solverstate"]
caffemodel = None
if "caffemodel" in msg:
caffemodel = msg["caffemodel"]
msg["error"] = self.start(solverstate, caffemodel)
msg["status"] = len(msg["error"]) == 0
self.transaction.send(msg)
def _pause(self):
msg = self.transaction.asyncRead()
msg["status"] = self.pause()
msg["iteration"] = self.iteration
if not msg["status"]:
msg["error"] = ["Could not pause a session in state " + str(self.state)]
self.transaction.send(msg)
self.save()
def _reset(self):
msg = self.transaction.asyncRead()
msg["status"] = self.reset()
msg["iteration"] = 0
if not msg["status"]:
msg["error"] = ["Could not reset a session in state " + str(self.state)]
self.transaction.send(msg)
self.save()
def _proceed(self):
msg = self.transaction.asyncRead()
if self.manager.isTraining():
msg["error"] = ["Could not proceed Session. There is already one running Session on this Server."]
msg["status"] = False
self.transaction.send(msg)
return
snapshot = None
if "snapshot" in msg:
snapshot = msg["snapshot"]
msg["error"] = self.proceed(snapshot)
msg["status"] = len(msg["error"]) == 0
self.transaction.send(msg)
def _msgSave(self):
msg = self.transaction.asyncRead()
error = []
ret = self.save(errors = error)
msg["status"] = ret
msg["error"] = error
self.transaction.send(msg)
def _takeSnapshot(self):
msg = self.transaction.asyncRead()
msg["status"] = self.snapshot()
if not msg["status"]:
msg["error"] = ["Could not take snapshot of session."]
self.transaction.send(msg)
def getRunLogFileName(self, basename=False):
""" Return the name of the logfile with session and run id.
"""
log_file = 'server_' + str(self.sid) + '.' + str(self.rid) + '.log'
if basename is True:
return log_file
log_file = os.path.join(self.getLogs(), log_file)
return log_file
def getParser(self):
""" Return the log parser of this session.
"""
if self.parser is None:
self.parser = Parser(None, Events.events)
self.parser.setLogging(True)
self.parser.printLogSignl.connect(self.Log, Qt.AutoConnection)
self.parser.addListener(self)
return self.parser
def getLogs(self):
""" Return the log directory.
"""
logs = os.path.join(self.directory, "logs")
if os.path.exists(logs) is False:
os.makedirs(logs)
return logs
def startParsing(self):
""" Create a parser and run it in a newly dispatched thread.
"""
self.parseOldLogs()
logs = self.getStream()
self.getParser().addLogStream(logs)
pool = SessionPool()
pool.addSession(self)
def parseOldLogs(self):
""" Parse all log files in the log directory.
"""
locked = self.lock.acquire()
if locked is False:
return
try:
if self.parse_old:
self.parse_old = False
log_files = {}
regex_filename = re.compile('[\d]+\.([\d]+)\.log$')
for entry in os.listdir(self.getLogs()):
filename_match = regex_filename.search(entry)
if filename_match:
# key files by run id
try:
run_id = int(filename_match.group(1))
log_files[run_id] = entry
except:
pass
log_list = []
for run_id in sorted(log_files.keys()):
log_file = os.path.join(self.getLogs(), log_files[run_id])
log_list.append(log_file)
con = Concatenator(log_list)
logs = con.concate()
for log in logs:
try:
self.getParser().addLogStream(log)
except Exception as e:
self.Log('Failed to parse log file ' + self.getLogFileName(True) + ": " + str(e), True)
except Exception as e:
self.Log('Failed to parse old log ' + str(e), True)
finally:
if locked:
self.lock.release()
def getLogFileName(self, basename=False):
""" Return the name of the logfile with session id.
"""
log_file = 'server_' + str(self.getSessionId()) + '.log'
if basename is True:
return log_file
log_file = os.path.join(self.getLogs(), log_file)
return log_file
def getStream(self):
""" Return the log stream of this session.
This is an iterator over stdout of the subprocess.
"""
if self.tee is not None:
return iter(self.tee.stdout.readline, '')
if self.proc is not None:
return iter(self.proc.stdout.readline, '')
return iter([])
def update(self, phase, row):
self.iteration = row['NumIters']
self.parssig.emit(phase, row)
def parsingFinished(self):
""" Called when the parser has processed all available streams.
"""
if self.parser_initialized:
self.logsig.emit("Parsing Finished", False)
if self.proc is not None:
# Wait for caffe process, kill tee and respond to return code
assert self.state is State.RUNNING
rcode = self.proc.wait()
self.proc = None
try:
self.tee.kill()
except Exception:
pass
self.tee = None
if rcode is 0:
self.setFinished()
else:
self.setState(State.FAILED)
self.Log('Session failed with return code ' + str(rcode), True)
self.parser_initialized = True
def registerKey(self, phase, key):
# TODO save the keys and send them on connection
self.keysig.emit(phase, key)
def handle(self, event, message, groups):
self.handlesig.emit(event, message, groups)
if event == 'OptimizationDone':
self.save()
self.setState(State.FINISHED)
self.proc = None
elif event == 'max_iter':
self.max_iter = int(groups[0])
# todo update max_iter
elif event == 'state_snapshot':
self.last_solverstate = groups[0]
#if self.parser_initialized:
self.snapshotsig.emit(self.last_solverstate)
elif event == 'model_snapshot':
self.last_caffemodel = groups[0]
def setFinished(self):
self.iteration = self.max_iter
self.setState(State.FINISHED)
def start(self, solverstate=None, caffemodel=None):
# TODO only one training per server
error = []
# (re-)write all session files
self.save(includeProtoTxt=True, errors=error)
if not os.path.exists(self.logs):
try:
os.makedirs(self.logs)
except OSError as e:
error.extend(['Failed to create Folder: ' + str(e)])
if self.rid is None:
self._parseSessionAndRunID()
if self.state is State.WAITING:
self.rid += 1
error.extend(self.prepairInternalPrototxt())
if len(error) > 0:
return error
caffe_bin = caffeVersions.getDefaultVersion().getBinarypath()
try:
self.getParser().setLogging(True)
# TODO set hardware trainOnHW
cmd = [
caffe_bin, 'train', '-solver',
os.path.join(self.directory, Paths.FILE_NAME_SOLVER)]
if solverstate is not None:
cmd.append('-snapshot')
cmd.append(str(solverstate))
if caffemodel is not None:
cmd.append('-weights')
cmd.append(str(caffemodel))
if self.manager.parent.trainOnHW > 0:
cmd.append('-gpu')
cmd.append(str(self.manager.parent.trainOnHW-1))
self.proc = Popen(
cmd,
stdout=PIPE,
stderr=STDOUT,
cwd=self.getSnapshotDirectory())
try:
self.tee = Popen(
['tee', '-a', self.getRunLogFileName()],
stdin=self.proc.stdout,
stdout=PIPE)
except Exception as e:
self.Log("Failed to start tee: " + str(e), error=True)
self.setState(State.RUNNING)
self.Log('Session ' + self.getRunLogFileName(True) + " was started.")
self.startParsing()
return []
except Exception as e:
return ["Failed to start session: " + str(e)]
else:
self.statesig.emit(self.state)
error.append("Failed to start session in state " + str(self.state))
return error
def pause(self):
if self.state is State.RUNNING:
if self.proc:
self.snapshot()
time.sleep(1)
try:
if self.proc:
self.proc.kill()
except Exception as e:
self.logsig.emit('Pausing session failed: ' + str(e), True)
try:
if self.tee:
self.tee.kill()
except Exception:
pass
self.proc = None
self.tee = None
self.last_solverstate = None
self.last_solverstate = self._getLastSnapshotFromSnapshotDirectory(True)
regex_iter = re.compile('iter_([\d]+)\.solverstate[\.\w-]*$')
iter_match = None
if self.last_solverstate is not None:
iter_match = regex_iter.search(self.last_solverstate)
if iter_match is not None:
self.iteration = int(iter_match.group(1))
# self.iterationChanged.emit() # implicit in _pause
self.setState(State.PAUSED)
self.logsig.emit("Session " + self.getRunLogFileName(True) + " was paused", False)
return True
return False
def snapshot(self):
if self.proc:
self.last_solverstate = None
try:
self.proc.send_signal(signal.SIGHUP)
except Exception as e:
self.Log('Failed to take snapshot: ' + str(e), True)
return False
self.Log('Snapshot was saved for session ' + self.getRunLogFileName(True))
return True
else:
self.statesig.emit(self.state)
self.Log('Could not take a snapshot in state ' + str(self.state))
return False
def proceed(self, snapshot=None):
if self.state is State.PAUSED:
if snapshot is None:
snapshot = self.getLastSnapshot()
caffe_bin = caffeVersions.getDefaultVersion().getBinarypath()
self.rid += 1
try:
self.getParser().setLogging(True)
cmd = [
caffe_bin, 'train', '-solver',
os.path.join(self.directory, Paths.FILE_NAME_SOLVER)]
cmd.append('-snapshot')
cmd.append(snapshot)
if self.manager.parent.trainOnHW > 0:
cmd.append('-gpu')
cmd.append(str(self.manager.parent.trainOnHW-1))
self.proc = Popen(
cmd,
stdout=PIPE,
stderr=STDOUT,
cwd=self.getDirectory())
try:
self.tee = Popen(
['tee', '-a', self.getRunLogFileName()],
stdin=self.proc.stdout,
stdout=PIPE)
except Exception as e:
self.Log("Failed to start tee: " + str(e), True)
self.setState(State.RUNNING)
self.Log('Session ' + self.getRunLogFileName(True) + " was proceeded.")
self.startParsing()
return []
except Exception as e:
return ["Failed to proceed session: " + str(e)]
else:
self.statesig.emit(self.state)
return ["Can't proceed a session in state " + str(self.state)]
def __ensureDirectory(self):
if os.path.exists(self.directory) is False:
try:
os.makedirs(self.directory)
return []
except Exception as e:
return ["Can't create Session Directory '" + self.directory + "': "+str(e)]
return []
def _msgFetchParserData(self):
msg = self.transaction.asyncRead()
msg["ParserRows"] = self.parserRows
msg["ParserKeys"] = self.parserKeys
msg["ParserHandle"] = self.parserHandle
msg["ParserLogs"] = self.parserLogs
self.transaction.send(msg)
def _msgLoadInternalNet(self):
msg = self.transaction.asyncRead()
try:
path = os.path.join(self.directory, Paths.FILE_NAME_NET_ORIGINAL)
file = open(path, 'r')
msg["NetInternal"] = file.read()
except IOError:
msg["status"] = False
msg["error"] = ["Failed to read " + path]
self.transaction.send(msg)
def _msgLoadDeployedNet(self):
msg = self.transaction.asyncRead()
try:
path = os.path.join(self.directory, Paths.FILE_NAME_NET_ORIGINAL)
file = open(path, 'r')
dn = DeployedNet(file.read())
msg["Net"] = dn.getProtoTxt()
except IOError:
msg["status"] = False
msg["error"] = ["Failed to read " + path]
self.transaction.send(msg)
def _msgLoadNetParameter(self):
msg = self.transaction.asyncRead()
msg["status"] = False
if "snapshot" in msg.keys():
path = msg["snapshot"]
path = os.path.join(self.directory, path)
if os.path.exists(path):
try:
with open(path, 'rb') as f:
msg["NetParams"] = f.read()
msg["status"] = True
except:
msg["error"] = ["Failed to load " + str(path)]
else:
msg["error"] = ["File not found " + str(path)]
else:
msg["error"] = ["No Snapshot provided"]
self.transaction.send(msg)
def _msgLoadCaffemodel(self):
msg = self.transaction.asyncRead()
msg["status"] = False
print("Load caffe model received.")
if "snapshot" in msg.keys():
path = msg["snapshot"]
print(path)
path = os.path.join(self.directory, path)
if os.path.exists(path):
try:
with open(path, 'rb') as f:
msg["caffemodel"] = f.read()
msg["status"] = True
except:
msg["error"] = ["Failed to load " + str(path)]
else:
msg["error"] = ["File not found " + str(path)]
else:
msg["error"] = ["No Snapshot provided"]
self.transaction.send(msg)
def reset(self):
self.pause()
for dirpath, dirnames, filenames in os.walk(self.directory, topdown=True):
for dirname in dirnames:
if os.path.join(dirpath, dirname) == self.logs:
try:
shutil.rmtree(self.logs)
except shutil.Error as e:
self.logsig.emit('Failed to delete logs folder: ' + str(e), self.getCallerId(), True)
for filename in filenames:
if filename.endswith(".solverstate") or filename.endswith(".caffemodel"):
if not filename == self.getPretrainedWeights():
try:
os.remove(os.path.join(dirpath, filename))
except OSError as e:
self.logsig.emit('Failed to delete ' + str(filename) + ': ' + str(e), self.getCallerId(), True)
if filename in ["net-internal.prototxt", "net-original.prototxt", "solver.prototxt"]:
try:
os.remove(os.path.join(dirpath, filename))
except OSError as e:
self.logsig.emit('Failed to delete ' + str(filename) + ': ' + str(e), self.getCallerId(), True)
break
self.iteration = 0
self.state = State.UNDEFINED
self._getState()
self.last_caffemodel = None
self.last_solverstate = None
self.save()
self.statesig.emit(self.state)
return True
Barista/backend/barista/session/session.py 0000664 ڊ0011117 00000117311 13265130053 021124 0 ustar klemms o0cvpr import os
import re
import shutil
import signal
import time
import json
import copy
from subprocess import Popen, PIPE, STDOUT
from threading import Lock
from datetime import datetime
from PyQt5.Qt import QObject
from PyQt5.QtCore import pyqtSignal
from backend.parser.concatenator import Concatenator
from backend.parser.parser_listener import ParserListener
from backend.parser.parser import Parser
from backend.barista.constraints.session_run.training import checkMinimumTrainingRequirements
from backend.barista.session.session_pool import SessionPool
from backend.barista.session.session_utils import *
from backend.barista.utils.logger import Log
from backend.barista.utils.logger import LogCaller
from backend.barista.deployed_net import DeployedNet
from backend.barista.session.session_common import SessionCommon
import backend.caffe.saver as saver
import backend.caffe.proto_info as info
import backend.caffe.dict_helper as helper
import backend.barista.caffe_versions as caffeVersions
class Session(QObject, LogCaller, ParserListener, SessionCommon):
""" A session is a caffe training process.
Objects of this class encapsulate the state of a process, control the
process (starting, pausing etc.), handle events and provide the session
output stream.
"""
# event signals
stateChanged = pyqtSignal(object)
stateDictChanged = pyqtSignal(object, bool)
iterationChanged = pyqtSignal()
snapshotAdded = pyqtSignal(object)
def __init__(self, project, directory=None, sid=None, parse_old=False, caffe_bin=None,
last_solverstate=None, last_caffemodel=None, state_dictionary=None):
super(Session, self).__init__()
self.caller_id = None
self.state = State.UNDEFINED
self.invalidErrorsList = []
self.sid = sid
self.directory = directory
self.rid = 0
self.project = project
self.logs = None
if self.directory is None:
if self.sid is None:
raise ValueError('Either directory or sid must be provided to create a session.')
self.directory = self.__createSessionDirectoryName()
self.logs = os.path.join(self.directory, 'logs')
else:
self.logs = os.path.join(directory, 'logs')
dir_sid, self.rid = self.__parseSessionId()
if self.sid is None:
self.sid = dir_sid
else:
Log.log('Provided sid and directory do not match ('+str(self.sid) + ' vs. ' + str(dir_sid) + '), ' +
'provided sid is used.', self.getCallerId())
self.parse_old = parse_old
self.caffe_bin = caffe_bin # overrides project caffe_root if necessary, i.e. if deployed to another system
self.pretrainedWeights = None
self.last_solverstate = last_solverstate
self.last_caffemodel = last_caffemodel
self.state_dictionary = state_dictionary # state as saved from the network manager, such it can be restored
self.start_time = self.__parseStartTime()
self.snapshot_dir = None
self.snapshot_prefix = None
self.proc = None
self.tee = None
self.parser = None
self.iteration = 0
self.max_iter = 1
self.parser_initialized = False
self.__getSettingsFromSessionFile()
if self.state_dictionary is not None:
self.__parseSettings(self.state_dictionary)
self.__solverFile = os.path.join(self.directory, Paths.FILE_NAME_SOLVER)
self.__netInternalFile = os.path.join(self.directory, Paths.FILE_NAME_NET_INTERNAL)
self.__netOriginalFile = os.path.join(self.directory, Paths.FILE_NAME_NET_ORIGINAL)
self.lock = Lock()
self.getSnapshotDirectory()
# comparison methods for the priorization of sessions in the thread pool
def __lt__(self, other):
if type(other) is not Session:
return True
return other.sid - self.sid < 0
def __gt__(self, other):
if type(other) is not Session:
return False
return other.sid - self.sid > 0
def __eq__(self, other):
if type(other) is not Session:
return False
return other.sid - self.sid == 0
def __le__(self, other):
if type(other) is not Session:
return True
return other.sid - self.sid <= 0
def __ge__(self, other):
if type(other) is not Session:
return False
return other.sid - self.sid >= 0
def __ne__(self, other):
if type(other) is not Session:
return True
return other.sid - self.sid != 0
def __createSessionDirectoryName(self):
""" Return a new session directory with the format YYYYMMDD_hh24mmss_SID
"""
directory = self.__getDateTimeString()
directory += '_'+str(self.sid)
return os.path.join(self.project.getSessionsDirectory(), directory)
def __ensureDirectory(self, directory):
""" Creates a directory if it does not exist.
"""
if directory == '':
return
if os.path.exists(directory) is False:
try:
os.makedirs(directory)
Log.log('Created directory: ' + directory, self.getCallerId())
except Exception as e:
Log.error('Failed to create directory '+directory+')'+str(e),
self.getCallerId())
def __getSettingsFromSessionFile(self):
filename = os.path.join(self.directory, baristaSessionFile(self.directory))
if os.path.isfile(filename):
with open(filename, 'r') as f:
res = json.load(f)
self.__parseSettings(res)
def checkFiles(self):
""" Check for the existence of the session directories and files.
"""
if os.path.exists(self.directory) is False:
Log.error('Session directory does not exists: '+self.directory,
self.getCallerId())
if os.path.exists(self.logs) is False:
Log.error('Log directory does not exists: '+self.logs,
self.getCallerId())
if os.path.exists(self.snapshot_dir) is False:
Log.error('Snapshot directory does not exists: '+self.snapshot_dir,
self.getCallerId())
if os.file.exists(caffeVersions.getVersionByName(self.project.getCaffeVersion())) is False:
Log.error('Caffe binary does not exists: ' +
self.project.getCaffeVersion(),
self.getCallerId())
def __parseSnapshotPrefix(self):
""" Parse the snapshot prefix from the solver file.
"""
if self.snapshot_prefix is None:
sf = os.path.join(self.getDirectory(), self.getSolver())
if os.path.isfile(sf):
self.snapshot_prefix = self.project.parseSnapshotPrefixFromFile(sf)
return
def getSnapshotPrefix(self):
""" Return the snapshot prefix which is used for snapshots of this
session.
"""
if not self.snapshot_prefix:
self.__parseSnapshotPrefix()
if not self.snapshot_prefix:
self.snapshot_prefix = ''
return self.snapshot_prefix
def getSnapshotExtension(self):
""" Return the file name extension for snapshot files.
The extensions differ between binaryproto and hdf5 formats.
"""
extension = None
if self.last_caffemodel:
# try to parse the extension from the last_caffemodel
regex_snapshot = re.compile('(\.caffemodel[\.\w-]*)')
snapshot_match = regex_snapshot.search(self.last_caffemodel)
if snapshot_match:
return snapshot_match.group(1)
else:
# try to parse the suffix from the last_solverstate
extension = self.getLastSnapshot()
if extension:
regex_snapshot = re.compile('\.solverstate([\.\w-]*)')
snapshot_match = regex_snapshot.search(extension)
if snapshot_match:
ext = snapshot_match.group(1)
if ext:
# append suffix
return '.caffemodel' + ext
return '.caffemodel'
def getSolverstateExtension(self):
""" Return the file name extension for solverstate files.
The extensions differ between binaryproto and hdf5 formats.
"""
extension = self.getLastSnapshot()
if extension:
regex_snapshot = re.compile('(\.solverstate[\.\w-]*)')
snapshot_match = regex_snapshot.search(extension)
if snapshot_match:
return snapshot_match.group(1)
return '.solverstate'
def getState(self):
""" Return the state of the session.
"""
if self.state == State.FAILED:
return self.state
if self.proc is not None:
if self.iteration == self.max_iter:
self.state = State.FINISHED
else:
self.state = State.RUNNING
else:
if len(checkMinimumTrainingRequirements(self)) > 0:
self.state = State.INVALID
elif self.iteration == self.max_iter:
self.state = State.FINISHED
elif len(self.getSnapshots()) > 0:
self.state = State.PAUSED
else:
self.state = State.WAITING
return self.state
def getErrorList(self):
return self.invalidErrorsList
def setState(self, state):
""" Set the state of the session and emit a stateChanged signal.
"""
self.state = state
self.stateChanged.emit(self.state)
def setErrorList(self, errorList):
self.invalidErrorsList = errorList
def setStateDict(self, stateDict):
self.state_dictionary = stateDict
self.stateDictChanged.emit(self, False)
return
def getSessionId(self):
""" Return the id of the session.
"""
return self.sid
def getRunId(self):
""" Return the run id of the session.
This id increases every time the process is started/proceeded.
It enables the user to distinguish between different runs of the
session.
"""
return self.rid
def start(self, solverstate=None, caffemodel=None):
""" Start the process.
Return
True if the process was started
False if the start failed
"""
if self.getState() is State.WAITING:
self.rid += 1
# (re-)write all session files
self.save(includeProtoTxt=True)
# check if the session has its own caffeRoot
caffeBin = self.caffe_bin
if not caffeBin:
# else take the project's caffeRoot path
caffeBin = caffeVersions.getVersionByName(self.project.getCaffeVersion()).getBinarypath()
try:
self.getParser().setLogging(True)
cmd = [caffeBin,
'train',
'-solver', self.getSolver()]
if solverstate:
cmd.append('-snapshot')
cmd.append(str(solverstate))
elif caffemodel:
cmd.append('-weights')
cmd.append(str(caffemodel))
self.proc = Popen(
cmd,
stdout=PIPE,
stderr=STDOUT,
cwd=self.getDirectory())
try:
self.tee = Popen(
['tee', '-a', self.getRunLogFileName()],
stdin=self.proc.stdout,
stdout=PIPE)
except Exception as e:
# continue without tee
Log.error('Failed to start tee: '+str(e),
self.getCallerId())
self.setState(State.RUNNING)
Log.log('Session '+self.getRunLogFileName(True) +
' was started', self.getCallerId())
self.__startParsing()
return True
except Exception as e:
# check if caffe root exists
Log.error('Failed to start session: '+str(e),
self.getCallerId())
if os.file.exists(caffeVersions.getVersionByName(self.project.getCaffeVersion()).getBinarypath()) is False:
Log.error('CAFFE_BINARY directory does not exists: ' +
caffe_bin +
'! Please set CAFFE_BINARY to run a session.',
self.getCallerId())
else:
Log.error(
'Could not start a session in state '+str(self.getState()),
self.getCallerId())
# self.setState(State.UNDEFINED)
return False
def pause(self):
""" Pause the process.
Return
True if the process was paused
False if the pause failed
"""
if self.getState() is State.RUNNING:
if self.proc:
self.snapshot()
# give the caffe process a second to save the state
time.sleep(1)
try:
if self.proc:
self.proc.kill()
except Exception as e:
Log.error('Pausing session failed: '+str(e))
try:
if self.tee:
self.tee.kill()
except Exception:
pass
self.proc = None
self.tee = None
self.last_solverstate = None
snap = self._getLastSnapshotFromSnapshotDirectory()
if snap is not None:
self.last_solverstate = os.path.basename(self._getLastSnapshotFromSnapshotDirectory())
regex_iter = re.compile('iter_([\d]+)\.solverstate[\.\w-]*$')
iter_match = regex_iter.search(self.last_solverstate)
if iter_match:
self.iteration = int(iter_match.group(1))
self.iterationChanged.emit()
self.setState(State.PAUSED)
Log.log('Session ' + self.getRunLogFileName(True) +
' was paused', self.getCallerId())
else:
self.setState(State.WAITING)
Log.log('Session ' + self.getRunLogFileName(True) +
' was halted', self.getCallerId())
self.save()
return True
else:
Log.error(
'Could not pause a session in state '+str(self.getState()),
self.getCallerId())
self.setState(State.UNDEFINED)
return False
def proceed(self, snapshot=None):
""" Continue training from the (last) snapshot.
Return
True if the process was continued
False if the continuation failed
"""
if self.getState() is State.PAUSED:
self.__ensureDirectory(self.getSnapshotDirectory())
self.__ensureDirectory(self.logs)
if snapshot is None:
snapshot = self.getLastSnapshot()
self.rid += 1
try:
self.getParser().setLogging(True)
self.proc = Popen([
caffeVersions.getVersionByName(self.project.getCaffeVersion()).getBinarypath(),
'train',
'-solver', self.getSolver(),
'-snapshot', snapshot],
stdout=PIPE,
stderr=STDOUT,
cwd=self.getDirectory())
try:
self.tee = Popen(
['tee', '-a', self.getRunLogFileName()],
stdin=self.proc.stdout,
stdout=PIPE)
except Exception as e:
# continue without tee
Log.error('Failed to start tee: '+str(e),
self.getCallerId())
self.setState(State.RUNNING)
Log.log('Session '+self.getRunLogFileName(True) +
' was proceeded', self.getCallerId())
self.__startParsing()
return True
except Exception as e:
# check if caffe root exists
Log.error('Failed to continue session: '+str(e),
self.getCallerId())
if os.file.exists(caffeVersions.getVersionByName(self.project.getCaffeVersion()).getBinarypath()) is False:
Log.error('CAFFE_BINARY directory does not exists: ' +
caffe_bin +
'! Please set CAFFE_BINARY to run a session.',
self.getCallerId())
elif self.getState() in (State.FAILED, State.FINISHED):
Log.error('Could not continue a session in state ' +
str(self.getState()), self.getCallerId())
return False
def snapshot(self):
""" Create a snapshot from the training state.
Return
True if the snapshot was created
False if the snapshot could not be created
"""
if self.getState() is State.RUNNING:
if self.proc:
self.last_solverstate = None
try:
self.proc.send_signal(signal.SIGHUP)
except Exception as e:
Log.error('Failed to take snapshot: '+str(e),
self.getCallerId())
return False
Log.log('Snapshot was saved for session ' +
self.getRunLogFileName(True)+'', self.getCallerId())
return True
else:
Log.error(
'Could not take a session snapshot in state ' +
str(self.getState()),
self.getCallerId())
return False
def setFinished(self):
""" Mark this session as finished.
"""
self.save()
self.setState(State.FINISHED)
self.proc = None
def getStream(self):
""" Return the log stream of this session.
This is an iterator over stdout of the subprocess.
"""
if self.tee:
return iter(self.tee.stdout.readline, '')
if self.proc:
return iter(self.proc.stdout.readline, '')
return iter([])
def getDirectory(self):
""" Return the session directory, usually a directory of the form
YYYYMMDD_hh24mmss_SID.
"""
return self.directory
def getLogs(self):
""" Return the log directory.
"""
if os.path.exists(self.logs) is False:
os.makedirs(self.logs)
return self.logs
def isRemote(self):
return False
def getSnapshotDirectory(self):
""" Return the snapshot directory.
"""
if self.snapshot_dir:
return self.snapshot_dir
snapshot_prefix = self.getSnapshotPrefix()
sdir = os.path.dirname(snapshot_prefix)
self.snapshot_dir = os.path.join(self.getDirectory(), sdir)
return self.snapshot_dir
def getSnapshots(self):
""" Return all snapshot files, keyed by iteration number.
"""
regex_snapshot = re.compile('iter_([\d]+)\.solverstate')
snaps = {}
if os.path.exists(self.getSnapshotDirectory()):
for entry in os.listdir(self.getSnapshotDirectory()):
snap_match = regex_snapshot.search(entry)
if snap_match:
try:
iter_num = int(snap_match.group(1))
snaps[iter_num] = entry
except Exception:
pass
return snaps
def getLastModel(self):
""" Return the name of the last saved caffe model.
"""
return self.last_caffemodel
def setLastModel(self, lcm):
self.last_caffemodel = lcm
def getPretrainedWeights(self):
return self.pretrainedWeights
def setPretrainedWeights(self, weights):
self.pretrainedWeights = weights
def setLastSnapshot(self, lss):
self.last_solverstate = lss
def getSolver(self, log=False):
""" Returns the solver prototxt file name.
When the log flag is set, a message will be sent to the logger console if the file does not exist.
"""
if log:
if not os.path.isfile(self.__solverFile):
Log.log("This sessions Solverfile: " + self.__solverFile + " does not exist.", self.caller_id)
return self.__solverFile
def getOriginalNetFile(self, log=False):
""" Returns the original net prototxt file name.
When the log flag is set, a message will be sent to the logger console if the file does not exist.
"""
if log:
if not os.path.isfile(self.__netOriginalFile):
Log.log("This sessions net file: " + self.__netOriginalFile + " does not exist.", self.caller_id)
return self.__netOriginalFile
def getInternalNetFile(self, log=False):
""" Returns the original net prototxt file name.
When the log flag is set, a message will be sent to the logger console if the file does not exist.
"""
if log:
if not os.path.isfile(self.__netInternalFile):
Log.log("This sessions net file: " + self.__netInternalFile + " does not exist.", self.caller_id)
return self.__netInternalFile
def readInternalNetFile(self):
""" Returns the contents of the internal net prototxt file. """
path = self.getInternalNetFile()
with open(path, 'r') as f:
contents = f.read()
return contents
def readDeployedNetAsString(self):
""" Returns the contents of the deployable net prototxt file. """
path = self.getInternalNetFile()
dn = DeployedNet(open(path).read())
return dn.getProtoTxt()
def readCaffemodelFile(self, snapshot):
""" Returns the contents of the .caffemodel file that belongs to snapshot.
snapshot: string
Filename without path of the snapshot file.
"""
path = os.path.join(self.getSnapshotDirectory(), snapshot)
with open(path, 'r') as f:
contents = f.read()
return contents
def getRunLogFileName(self, basename=False):
""" Return the name of the logfile with session and run id.
"""
log_file = self.project.getProjectName()+'_'+str(self.getSessionId())+'.'+str(self.getRunId())+'.log'
if basename is True:
return log_file
log_file = os.path.join(self.getLogs(), log_file)
return log_file
def getLogFileName(self, basename=False):
""" Return the name of the logfile with session id.
"""
log_file = self.project.getProjectName()+'_'+str(self.getSessionId())+'.log'
if basename is True:
return log_file
log_file = os.path.join(self.getLogs(), log_file)
return log_file
def getLogId(self):
""" Return the id of the logfile. For local sessions same as the name of the logfile.
"""
return self.getLogFileName(True)
def parseOldLogs(self):
""" Parse all log files in the log directory.
"""
locked = self.lock.acquire()
if locked is False:
return
try:
if self.parse_old:
self.parse_old = False
log_files = {}
regex_filename = re.compile('[\d]+\.([\d]+)\.log$')
for entry in os.listdir(self.getLogs()):
filename_match = regex_filename.search(entry)
if filename_match:
# key files by run id
try:
run_id = int(filename_match.group(1))
log_files[run_id] = entry
except:
pass
log_list = []
for run_id in sorted(log_files.keys()):
log_file = os.path.join(self.getLogs(), log_files[run_id])
log_list.append(log_file)
con = Concatenator(log_list)
logs = con.concate()
for log in logs:
try:
self.getParser().addLogStream(log)
except Exception as e:
Log.error('Failed to parse log file '+self.getLogFileName(True)+": "+str(e), self.getCallerId())
except Exception as e:
Log.error('Failed to parse old log '+str(e), self.getCallerId())
finally:
if locked:
self.lock.release()
def __startParsing(self):
""" Create a parser and run it in a newly dispatched thread.
"""
self.parseOldLogs()
logs = self.getStream()
self.getParser().addLogStream(logs)
pool = SessionPool()
pool.addSession(self)
def getParser(self):
""" Return the log parser of this session.
"""
if self.parser is None:
self.parser = Parser(None, Events.events, self.getCallerId())
self.parser.addListener(self)
return self.parser
def getIteration(self):
""" Return the current training iteration of this session.
"""
return self.iteration
def getMaxIteration(self):
""" Return the maximum training iteration of this session.
"""
return self.max_iter
def setMaxIteration(self, maxIteration):
""" Set the maximum training iteration of this session.
"""
if maxIteration > 0:
self.max_iter = maxIteration
self.stateChanged.emit(self.getState())
def setParserInitialized(self):
""" Should be called after the parser finished the inital parsing of
log files.
"""
#if self.parser_initialized is False:
# self.snapshotAdded.emit(self.last_solverstate)
self.parser_initialized = True
def isParserInitialized(self):
return self.parser_initialized
def delete(self):
""" Delete the session directory and disconnect signals.
"""
self.pause()
try:
shutil.rmtree(self.getDirectory())
except Exception as e:
Log.error('Could not remove session directory: '+str(e),
self.getCallerId())
try:
self.stateChanged.disconnect()
self.iterationChanged.disconnect()
self.snapshotAdded.disconnect()
self.project.deleteSession.emit(self.sid)
except Exception as e:
pass
Log.removeCallerId(self.caller_id, False)
# LogCaller
def getCallerId(self):
""" Return the unique caller id for this session
"""
if self.caller_id is None:
self.caller_id = Log.getCallerId(self.getLogFileName(True))
return self.caller_id
# ParserListener
def update(self, phase, row):
""" Called when the parser has parsed a new record.
"""
self.iteration = row['NumIters']
self.iterationChanged.emit()
def handle(self, event, message, groups):
""" Called when the parser has parsed a registered event.
"""
if event == 'OptimizationDone':
self.setFinished()
elif event == 'max_iter':
self.max_iter = int(groups[0])
elif event == 'state_snapshot':
self.last_solverstate = groups[0]
#if self.parser_initialized:
self.snapshotAdded.emit(self.last_solverstate)
elif event == 'model_snapshot':
self.last_caffemodel = groups[0]
def parsingFinished(self):
""" Called when the parser has processed all available streams.
"""
if self.proc is not None:
# Wait for caffe process, kill tee and respond to return code
assert self.state is State.RUNNING
rcode = self.proc.wait()
self.proc = None
try:
self.tee.kill()
except Exception:
pass
self.tee = None
if rcode is 0:
self.setFinished()
else:
self.setState(State.FAILED)
Log.error('Session failed with return code '+str(rcode),
self.getCallerId())
self.setParserInitialized()
def hasStateDict(self):
return self.state_dictionary is not None
def save(self, includeProtoTxt = False):
"""Saves the current session to a json file. If includeProtoTxt is True, prototxt files are saved as well."""
toSave = {"SessionState": self.state, "Iteration": self.iteration, "MaxIter": self.max_iter}
toSave["ProjectID"] = self.project.projectId
self.__ensureDirectory(self.directory)
Log.log("Saving current Session status to disk.", self.getCallerId())
if self.last_solverstate:
toSave["LastSnapshot"] = self.last_solverstate
if self.getPretrainedWeights():
toSave["PretrainedWeights"] = self.getPretrainedWeights()
if self.state_dictionary:
serializedDict = copy.deepcopy(self.state_dictionary)
if includeProtoTxt:
if "solver" in self.state_dictionary:
solver = self.buildSolverPrototxt()
with open(self.getSolver(log=False), 'w') as f:
f.write(solver)
else:
Log.error("Could not save a solver prototxt file, because no solver settings are defined.", self.getCallerId())
if "network" in serializedDict:
if includeProtoTxt:
net = self.buildNetPrototxt(internalVersion=False)
with open(self.getOriginalNetFile(log=False), 'w') as f:
f.write(net)
net = self.buildNetPrototxt(internalVersion=True)
with open(self.getInternalNetFile(log=False), 'w') as f:
f.write(net)
if "layers" in serializedDict["network"]:
layers = serializedDict["network"]["layers"]
for id in layers:
del layers[id]["type"]
else:
Log.error("Could not save the network state because no state was defined.", self.getCallerId())
toSave["NetworkState"] = serializedDict
with open(baristaSessionFile(self.directory), "w") as f:
json.dump(toSave, f, sort_keys=True, indent=4)
def _modifyNetDictionaryToInternalVersion(self, net):
""" Take an original net dictionary and apply all changes necessary for
the internal version.
Mainly, this will change all relative paths to be relative to a session
folder.
Hint: This will change net itself. You might want to create a (deep)
copy of the dictionary before calling this method.
"""
# define all layer parameters that can contain (relative) paths
# (use a dot to separate nested parameters)
layerParamsContainingPaths = [
"data_param.source",
"hdf5_data_param.source",
"image_data_param.source",
"window_data_param.source",
"data_param.mean_file",
"hdf5_output_param.file_name",
"image_data_param.mean_file",
"window_data_param.mean_file",
"transform_param.mean_file"]
# evaluate layer after layer
h = helper.DictHelper(net)
for layerId, layer in net.get("layers", {}).iteritems():
# evaluate parameter after parameter
for paramKey in layerParamsContainingPaths:
# if the current layer does have the current parameter..
if h.layerParameterIsSet(layerId, paramKey):
paramValue = h.layerParameter(layerId, paramKey)
# ..and its value is a relative path: modify it
if paramValue is not None and not os.path.isabs(paramValue):
if paramKey != "hdf5_data_param.source":
newPath = os.path.join(
os.pardir,
os.path.join(os.pardir, paramValue)
)
else:
newFilename = str(layerId) + ".txt"
newPath = os.path.join(os.curdir, newFilename)
h.setLayerParameter(layerId, paramKey, newPath)
return net
def buildSolverPrototxt(self):
""" Load the current solver dictionary and return the corresponding
message object.
:return: A solver message object.
"""
if not self.state_dictionary:
return None
elif "solver" not in self.state_dictionary:
return None
else:
solver = saver.saveSolver(self.state_dictionary["solver"])
return solver
def buildNetPrototxt(self, internalVersion=False):
""" Load the current net dictionary and return the corresponding
message object.
:param internalVersion: Iff true, the loaded net will be modified to
the internal version.
:return: A net message object.
"""
currentState = self.state_dictionary
netDictionary = None
if currentState:
netDictionary = currentState["network"]
if internalVersion:
netDictionary = self._modifyNetDictionaryToInternalVersion(copy.deepcopy(netDictionary))
solver = saver.saveNet(netDictionary)
return solver
def __parseSettings(self, settings):
if settings:
if "SessionState" in settings:
self.state = settings["SessionState"]
self.__previousState = self.state
if "Iteration" in settings:
self.iteration = settings["Iteration"]
if "solver" in settings:
if "max_iter" in settings["solver"]:
self.max_iter = settings["solver"]["max_iter"]
elif "MaxIter" in settings:
self.max_iter = settings["MaxIter"]
if "LastSnapshot" in settings:
self.last_solverstate = settings["LastSnapshot"]
if "PretrainedWeights" in settings:
self.setPretrainedWeights(settings["PretrainedWeights"])
if "NetworkState" in settings:
self.state_dictionary = settings["NetworkState"]
layers = self.state_dictionary["network"]["layers"]
for id in layers:
if "parameters" in layers[id]:
if "type" in layers[id]["parameters"]:
typename = layers[id]["parameters"]["type"]
layers[id]["type"] = info.CaffeMetaInformation().getLayerType(typename)
# layers[id]["type"] = info.CaffeMetaInformation().availableLayerTypes()[typename]
solver = self.state_dictionary["solver"]
if solver:
if "snapshot_prefix" in solver:
self.snapshot_prefix = solver["snapshot_prefix"]
def updateMaxIterFromStateDict(self):
if self.state_dictionary:
if "solver" in self.state_dictionary:
if "max_iter" in self.state_dictionary["solver"]:
self.max_iter = self.state_dictionary["solver"]["max_iter"]
elif "MaxIter" in self.state_dictionary:
self.max_iter = self.state_dictionary["MaxIter"]
def __parseSessionId(self):
""" Return a tuple (session_id, run_id). The ids are parsed from the
directory. session_id is the id of the session and run_id the highest
found run id.
"""
regex_sid = re.compile('[\d]{8}_[\d]{6}_([\d]+)')
sid_match = regex_sid.search(self.directory)
session_id = None
if sid_match:
try:
session_id = int(sid_match.group(1))
except:
pass
run_id = 0
regex_rid = re.compile('([\d]+)\.([\d]+)\.log')
if os.path.exists(self.logs):
for entry in os.listdir(self.logs):
rid_match = regex_rid.search(entry)
if rid_match:
try:
_run_id = int(rid_match.group(2))
if run_id:
if _run_id > run_id:
run_id = _run_id
else:
run_id = _run_id
if session_id is None:
session_id = int(rid_match.group(1))
except:
pass
return session_id, run_id
def __parseStartTime(self):
""" Parse the start time with second precision from the directory name.
Return the parsed datetime.
"""
regex_time = re.compile(
'([\d]{4})([\d]{2})([\d]{2})_([\d]{2})([\d]{2})([\d]{2})')
time_match = regex_time.search(self.directory)
time = datetime.now()
if time_match:
try:
year = int(time_match.group(1))
month = int(time_match.group(2))
day = int(time_match.group(3))
hour = int(time_match.group(4))
minute = int(time_match.group(5))
second = int(time_match.group(6))
time = time.replace(year=year, month=month, day=day, hour=hour,
minute=minute, second=second)
except Exception:
pass
return time
def __getDateTimeString(self):
""" Return the current datetime in the format YYYYMMDD_hh24mmss
"""
now = datetime.now()
date_string = ''
date_string += str(now.year)
date_string += str(now.month).zfill(2)
date_string += str(now.day).zfill(2)
date_string += '_'
date_string += str(now.hour).zfill(2)
date_string += str(now.minute).zfill(2)
date_string += str(now.second).zfill(2)
return date_string
def reset(self):
self.pause()
for dirpath, dirnames, filenames in os.walk(self.directory, topdown=True):
for dirname in dirnames:
if os.path.join(dirpath, dirname) == self.logs:
try:
shutil.rmtree(self.logs)
except shutil.Error as e:
Log.error('Failed to delete logs folder: ' + str(e), self.getCallerId())
for filename in filenames:
if filename.endswith(".solverstate") or filename.endswith(".caffemodel"):
if not filename == self.getPretrainedWeights():
try:
os.remove(os.path.join(dirpath, filename))
except OSError as e:
Log.error('Failed to delete ' + str(filename) + ': ' + str(e), self.getCallerId())
if filename in ["net-internal.prototxt", "net-original.prototxt", "solver.prototxt"]:
try:
os.remove(os.path.join(dirpath, filename))
except OSError as e:
Log.error('Failed to delete ' + str(filename) + ': ' + str(e), self.getCallerId())
break
self.iteration = 0
self.iterationChanged.emit()
self.state = State.UNDEFINED
self.setState(self.getState())
self.setLastModel(None)
self.setLastSnapshot(None)
self.project.resetSession.emit(self.getSessionId())
self.save() Barista/backend/barista/session/session_common.py 0000664 ڊ0011117 00000006361 13265130053 022476 0 ustar klemms o0cvpr import os
import re
from abc import abstractmethod
class SessionCommon():
def __init__(self):
self.lastSolverState = None
self.directory = None
@abstractmethod
def getLogs(self):
pass
@abstractmethod
def getSnapshotDirectory(self):
pass
def getLastSnapshot(self):
""" Return the last snapshot/solverstate for this session.
The last snapshot name is searched in the log files.
"""
if self.last_solverstate:
if os.path.isfile(os.path.join(self.directory, self.last_solverstate)):
return self.last_solverstate
self.last_solverstate = self._getLastSnapshotFromLogFiles()
if self.last_solverstate:
if os.path.isfile(os.path.join(self.directory, self.last_solverstate)):
return self.last_solverstate
self.last_solverstate = self._getLastSnapshotFromSnapshotDirectory()
if self.last_solverstate:
if os.path.isfile(os.path.join(self.directory, self.last_solverstate)):
return self.last_solverstate
return None
def _getLastSnapshotFromLogFiles(self):
""" Try to find the last snapshot name in the log file.
Return the name of the solverstate file if it was found.
"""
# get all log files
log_files = {}
regex_filename = re.compile('[\d]+\.([\d]+)\.log$')
for entry in os.listdir(self.getLogs()):
filename_match = regex_filename.search(entry)
if filename_match:
# key files by run id
try:
run_id = int(filename_match.group(1))
log_files[run_id] = entry
except:
pass
last_solverstate = None
for run_id in reversed(sorted(log_files.keys())):
with open(os.path.join(self.getLogs(), log_files[run_id])) as f:
# find the last snapshot in the file
regex_snapshot = re.compile(
'Snapshotting solver state to (?:binary proto|HDF5) file (.+\.solverstate[\.\w-]*)')
for line in f:
snapshot_match = regex_snapshot.search(line)
if snapshot_match:
last_solverstate = snapshot_match.group(1)
if last_solverstate:
return last_solverstate
def _getLastSnapshotFromSnapshotDirectory(self, basename=False):
""" Try to find the last snapshot in the snapshot directory.
Return the name of the solverstate file if it was found.
"""
solver_state = None
max_iter = -1
regex_iter = re.compile('iter_([\d]+)\.solverstate[\.\w-]*$')
for entry in os.listdir(self.getSnapshotDirectory()):
iter_match = regex_iter.search(entry)
if iter_match:
try:
iter_id = int(iter_match.group(1))
if iter_id > max_iter:
max_iter = iter_id
solver_state = entry
except:
pass
if solver_state:
if basename:
return solver_state
return os.path.join(self.getSnapshotDirectory(), solver_state)
Barista/backend/barista/session/session_pool.py 0000664 ڊ0011117 00000006105 13265130053 022153 0 ustar klemms o0cvpr import multiprocessing
import sys
if sys.version_info[0] == 2:
from Queue import PriorityQueue, Empty
else:
from queue import PriorityQueue, Empty
from threading import Thread
class Singleton(type):
"""This metaclass is used to provide the singleton pattern in a generic way.
See http://stackoverflow.com/a/6798042 for source and further explanation.
TODO outsource this class to use the same pattern in the complete project?
"""
_instances = {}
def __call__(cls, *args, **kwargs):
if cls not in cls._instances:
cls._instances[cls] = super(Singleton, cls).__call__(*args, **kwargs)
return cls._instances[cls]
class SessionPool:
""" A simple thread pool implementation for parsing log files of sessions.
A job is added by adding a session.
Jobs are ordered by the session_id(priority), so new sessions have a higher
priority.
The pool allocates and starts threads on demand. It allocates no more then
CPU_COUNT - 2 threads, but at least 1.
"""
__metaclass__ = Singleton
def __init__(self):
self.sessions = PriorityQueue()
self.pool = []
self.__start_jobs = False
self.MAX_THREADS = multiprocessing.cpu_count() - 2
if self.MAX_THREADS <= 0:
self.MAX_THREADS = 1
def addSession(self, session):
""" Add a session to to queue and start a thread.
"""
self.sessions.put(session)
if self.__start_jobs:
self.__startJob()
def activate(self, emptyJob):
""" Activates this pool by starting threads.
"""
self.__start_jobs = True
self.emptyJob = emptyJob
qs = self.sessions.qsize()
starts = min(qs, self.MAX_THREADS)
for i in range(0, starts):
self.__startJob()
# private methods
def __startJob(self):
""" Remove inactive threads from the pool and create a new for the new
job.
"""
if self.__start_jobs is False:
return
# clean thread pool
jc = len(self.pool)
for ri in range(jc, 0, -1):
i = ri - 1
thread = self.pool[i]
if thread is not None:
if thread.is_alive() is False:
self.pool.pop(i)
# create a new thread
jc = len(self.pool)
if jc < self.MAX_THREADS:
t = Thread(target=self.__executeJob)
t.start()
self.pool.append(t)
def __executeJob(self):
""" The target method of a thread.
Poll for sessions and execute their parser.
"""
try:
while True:
session = self.sessions.get(True, 1)
if session:
session.parseOldLogs()
parser = session.getParser()
parser.parseLog()
self.sessions.task_done()
else:
break
except Empty:
if self.emptyJob:
self.emptyJob()
return
except Exception as e:
print(str(e))
Barista/backend/barista/session/session_utils.py 0000664 ڊ0011117 00000003075 13265130053 022345 0 ustar klemms o0cvpr class State:
""" Pseudo Enumeration for the states a session could be in.
"""
UNDEFINED = 0
WAITING = 1
RUNNING = 2
PAUSED = 3
FAILED = 5
FINISHED = 6
INVALID = 7
NOTCONNECTED = 8
def baristaSessionFile(directory):
import os
""" Returns the filename of the config-json file in the given directory """
return os.path.join(directory, "sessionstate.json")
class Paths:
# class "constants" defining common file names inside of a session
FILE_NAME_SOLVER = "solver.prototxt"
FILE_NAME_NET_ORIGINAL = "net-original.prototxt"
FILE_NAME_NET_INTERNAL = "net-internal.prototxt"
FILE_NAME_SESSION_JSON = "sessionstate.json"
class Events:
import re
events = {
# exceptions
'FileNotFound': re.compile('Check failed: mdb_status == 0 \(([\d]+) vs\. 0\) No such file or directory'),
'OutOfGPU': re.compile('Check failed: error == cudaSuccess \(([\d]+) vs\. 0\) out of memory'),
'NoSnapshotPrefix': re.compile('Check failed: param_\.has_snapshot_prefix\(\) In solver params, snapshot is specified but snapshot_prefix is not'),
# session finished
'OptimizationDone': re.compile('Optimization Done'),
# iterations
'max_iter': re.compile('max_iter:[\s]+([\d]+)'),
# snapshots
'state_snapshot': re.compile('Snapshotting solver state to (?:binary proto|HDF5) file (.+\.solverstate[\.\w-]*)'),
'model_snapshot': re.compile('Snapshotting to (?:binary proto|HDF5) file (.+\.caffemodel[\.\w-]*)')
} Barista/backend/barista/utils/__init__.py 0000664 ڊ0011117 00000000000 13265130053 020637 0 ustar klemms o0cvpr Barista/backend/barista/utils/db_util.py 0000664 ڊ0011117 00000012473 13265130053 020543 0 ustar klemms o0cvpr import os
import hashlib
from backend.barista.utils.logger import *
"""
This file contains functions that deal with the path handling and hashing of database files
@author: j_stru18
"""
"""
Logger ID used in functions below.
"""
DBUTIL_LOGGER_ID = Log.getCallerId("dbutil")
"""
returns a hash value by calling a hash function for the opened file of the given path
Warning: This is not meant to be used for HDF5TXT files. You need to use getHdf5List() and call getMultipleHash
with the result.
@author ..., f_prob02, j_stru18, l_hu0002
@param db_path: a path to a database file, for example a data.mdb
@return a hash value for the hashed database as int
@pre dbPath is a valid path
"""
def getHash(dbPath):
hashObject = hashlib.sha256()
with open(dbPath, "rb") as openDB:
for chunk in iter(lambda: openDB.read(2**15), b""):
hashObject.update(chunk)
return int(hashObject.hexdigest(), 16)
"""
hashes all files in the given paths
@author j_stru18
@param dbPaths a list of paths
@return a hash value for all files in the given paths as int
"""
def getMultipleHash(dbPaths):
hashValue = 0
for path in dbPaths: # hash all the old .h5 and .hdf5 files in HDF5TXT
if os.path.exists(path):
hashValue += getHash(path)
return hashValue
"""
returns the sha256 hash value of a single string
@author j_stru18
@param inputStr : the string that has to be hashed
@return : the hashValue of the string
"""
def getStringHash(inputStr):
hashObject = hashlib.sha256()
hashObject.update(inputStr)
return int(hashObject.hexdigest(), 16)
"""
returns a list containing the path to a lmdb, if this file exists
@author j_stru18
@param path : a string that is supposed to be a valid path for a lmdb
@return a list that contains the given path
"""
def getLMDBList(path):
pathList = []
if os.path.exists(path):
pathList.append(path)
return pathList
"""
@author j_stru18
@param path : a string that is supposed to be a valid path for a leveldb (directory path containing a CURRENT file)
@return a list that contains all the .sst and .ldb files in the directory
"""
def getLEVELDBList(directoryPath):
pathList = []
if os.path.exists(os.path.join(directoryPath, "CURRENT")):
for db_file in os.listdir(directoryPath):
if db_file.endswith(".sst") or db_file.endswith(".ldb"):
pathList.append(os.path.join(directoryPath, db_file))
return pathList
"""
returns a list containing the paths to all .h5 or .hdf5 files that are specified in a HDF5TXT file, if this file
exists
@author j_stru18
@ensure: if filePath points to a valid file, it has to a textfile
@param filePath : a string that is supposed to be a valid path for a HDF5TXT file
@return a list that contains all .h5 and .hdf5 files in the HDF5TXT
"""
def getHdf5List(filePath):
pathList = []
if os.path.exists(filePath):
lines = [line.rstrip('\n') for line in open(filePath)]
for line in lines:
if line.endswith(".h5") or line.endswith(".hdf5"):
if os.path.exists(line): # absolute path in txt
pathList.append(line)
elif os.path.exists(os.path.normpath(os.path.join(filePath, line))): # path relative to HDF5 location
pathList.append(os.path.normpath(os.path.join(filePath, line)))
else:
Log.error("HDF5 textfile contained invalid path {}".format(line), DBUTIL_LOGGER_ID)
return pathList
"""
returns a list containing all lines of the file that filePath points at
@author: j_stru18
@param filePath : a string that is supposed to be a path for a txt file
@return: a list that contains all lines of that file
"""
def getLinesAsList(filePath):
pathList = []
if os.path.exists(filePath):
pathList = [line for line in open(filePath)]
return pathList
"""
Given the name of a file, this method returns its type
@author j_stru18
@param filename a string that represents the file's name
@return its type as a string
"""
def getType(filename):
type = ""
if filename == "CURRENT":
type = "LEVELDB"
else:
extension = filename[-4:]
if extension == ".mdb":
type = "LMDB"
elif extension == ".ldb":
type = "LEVELDB"
elif extension == ".txt":
type = "HDF5TXT"
elif filename[-5:] == ".hdf5":
type = "HDF5"
elif filename[-3:] == ".h5":
type = "HDF5"
return type
"""
Given the path and type of a file, this method returns the paths to all files that this db file uses to store data.
@author: j_stru18
@param filePath : the path to a db file as string
@param type : the db file's type as string
@return : all paths of this db file that are used to store data
"""
def getDBPathsByType(filePath, type):
dbpaths = []
if type == 'LMDB':
filePath = os.path.normpath(os.path.join(filePath, "data.mdb"))
dbpaths = getLMDBList(filePath)
elif type == 'LEVELDB':
directoryPath = os.path.normpath(filePath)
dbpaths = getLEVELDBList(directoryPath)
elif type == 'HDF5TXT':
filePath = os.path.normpath(filePath)
dbpaths = getHdf5List(filePath)
elif type == 'HDF5':
dbpaths.append(filePath)
else:
Log.error("File has the wrong type: {}".format(type), DBUTIL_LOGGER_ID)
return dbpaths
Barista/backend/barista/utils/logger.py 0000664 ڊ0011117 00000015237 13265130053 020401 0 ustar klemms o0cvpr from PyQt5.QtCore import Qt
from PyQt5.Qt import QObject
import PyQt5.QtCore as QtCore
import inspect
import datetime
import time as tim
'''
Model/Helper-Classes
'''
class MessageType:
def __init__(self, id, description):
self.typeId = id
if(id == -1): # DockElementConsole.ALL
self.color = Logger.COLORS[Logger.TEXT.typeId] # not existiing now
else:
self.color = Logger.COLORS[id]
self.description = description
class LogLine:
def __init__(self, line, caller, msgType, time):
self.line = line
self.caller = caller
self.msgType = msgType
self.time = time
class Caller:
def __init__(self, id, description, customColor):
# if no id is given, default to 0
if not id:
id = 0
self.callerId = id
id += 2 # color offset for reserved colors black and red
if id >= len(Logger.COLORS) or not customColor:
self.color = Logger.COLORS[
Logger.TEXT.typeId]
else:
self.color = Logger.COLORS[id]
self.description = description
self.used = False
def setUsed(self):
self.used = True
class LogCaller:
""" A interface for users of the Logger class.
If a class implements this interface, it can use the Logger methods log and
error without delivering the callerId
"""
def getCallerId(self):
pass
class Logger(QObject):
"""
TODO: Selection is removed on insertion, fix needed?
TODO: Logfile schreiben
"""
newLine = QtCore.pyqtSignal(object)
sigRefreshGui = QtCore.pyqtSignal(object)
sigRefreshCallers = QtCore.pyqtSignal(object)
# Color Constants
COLORS = [Qt.red, Qt.black, Qt.blue, Qt.green, Qt.yellow]
# Type Constants, used for color and combobox index
# initalized at the bottom of this module
ERROR = None
TEXT = None
ALL = None
def __init__(self):
super(Logger, self).__init__()
self.__loglines = []
self.__callerIdCount = 0
self.__removedCallers = []
self.__callers = {}
self.__guiConsole = None
self.__filePath = None
self.__defaultCaller = Caller(-1, "Caller could not be identified", False)
def getCallerFromIdWithFallback(self, callerId):
if callerId is None or callerId not in self.__callers:
return self.__defaultCaller
else:
return self.__callers[callerId]
def log(self, line, callerId=None):
'''
use to log simple text
'''
# inspired by http://stackoverflow.com/a/7272464/2129327
if callerId is None:
try:
caller = inspect.currentframe().f_back.f_locals['self']
if caller:
log_id = getattr(caller, "getCallerId()", None)
if callable(log_id):
callerId = caller.getCallerId()()
except Exception:
# TODO: Warn properly that something went wrong
print('From unknown callerID: ' + line)
pass
self.appendLine(line, callerId)
def error(self, line, callerId=None):
'''
use to log errors
'''
# inspired by http://stackoverflow.com/a/7272464/2129327
if callerId is None:
try:
caller = inspect.currentframe().f_back.f_locals['self']
if caller:
log_id = getattr(caller, "getCallerId()", None)
if callable(log_id):
callerId = caller.getCallerId()()
except Exception:
pass
self.appendLine(line, callerId, Logger.ERROR)
def appendLine(self, line, callerId, msgType=None):
'''
appends a sing line to the logger and if existing to console
and file
line: String , the string to append
callerId: Int, the id created by getCallerId
msgType: MessageType, An Object indicating which Type this logline is
'''
if(msgType is None):
msgType = Logger.TEXT
ts = tim.time()
st = datetime.datetime.fromtimestamp(ts).strftime("%H:%M:%S")
caller = self.getCallerFromIdWithFallback(callerId)
caller.setUsed()
logline = LogLine(line, caller, msgType, st)
self.__loglines.append(logline)
# notify all consoles etc about new line
self.newLine.emit(logline)
if self.__filePath is not None:
self.__guiConsole.appendLineToConsole(logline)
def appendLines(self, lines, callerId):
'''
TODO: Possible optimization for big amount of data, like logfiles
'''
for line in lines:
self.appendLine(line, callerId)
def getCallerId(self, description, customColor=False):
'''
creates a new Caller id which identifies the
using object to group their log messages
'''
if(self.__removedCallers): # list not empty
callerId = self.__removedCallers.pop()
else:
callerId = self.__callerIdCount
self.__callerIdCount += 1
caller = Caller(callerId, description, customColor)
self.__callers[callerId] = caller
self.refreshCallers()
return callerId
def removeCallerId(self, callerId, keepLines=True):
if(callerId in self.__removedCallers):
return
self.__removedCallers.append(callerId)
self.__callers.pop(callerId, None)
if(keepLines == False):
filterLines = lambda logline: logline.caller.callerId != callerId
self.__loglines = list(filter(filterLines, self.__loglines))
self.refreshGui()
else:
self.refreshCallers()
def setFile(self, filePath):
self.__filePath = filePath
def refreshGui(self):
'''
refreshs/validates the whole console and filter values
'''
self.refreshConsole()
self.refreshCallers()
def refreshConsole(self):
'''
refreshs/validates the whole console
'''
self.sigRefreshGui.emit(self.__loglines)
def refreshCallers(self):
'''
refreshs/validates the filter values/callers
'''
self.sigRefreshCallers.emit(self.__callers)
def __getPrefix(self, caller, time):
'''
the prefix that have to be added
'''
prefix = ""
prefix = "[" + time + ", " + caller.description + "]"
return prefix
# Add Constant Values to Logger
Logger.ERROR = MessageType(0, "ERROR")
Logger.TEXT = MessageType(1, "TEXT")
Logger.ALL = MessageType(-1, "ALL")
# "Singelton" instance
Log = Logger()
Barista/backend/barista/utils/settings.py 0000664 ڊ0011117 00000000376 13265130053 020760 0 ustar klemms o0cvpr from PyQt5.QtCore import QSettings
def applicationQSetting():
""" Returns the default QSettings-Object for this application """
# return QSettings(QSettings.IniFormat,QSettings.UserScope, "wwu", "Barista")
return QSettings("wwu", "Barista")
Barista/gui/main_window/docks/__init__.py 0000664 ڊ0011117 00000000000 13265130053 020665 0 ustar klemms o0cvpr Barista/gui/main_window/docks/dock.py 0000664 ڊ0011117 00000001432 13265130053 020060 0 ustar klemms o0cvpr # To change this license header, choose License Headers in Project Properties.
# To change this template file, choose Tools | Templates
# and open the template in the prototxt_editor.
from PyQt5 import QtGui, QtCore, QtWidgets
import functools
class DockElement(QtWidgets.QDockWidget):
def __init__(self, mainWindow, title):
QtWidgets.QDockWidget.__init__(self, title)
palette = self.palette()
palette.setColor(QtGui.QPalette.Window, QtCore.Qt.white)
self.setPalette(palette)
self.setAutoFillBackground(1)
self.adjustSize()
self.setObjectName(title)
# Connect the visibility changed event to the view manager
self.visibilityChanged.connect(functools.partial(mainWindow.viewManager.onVisibilityChange)) Barista/gui/main_window/docks/dock_active_layers.py 0000664 ڊ0011117 00000053750 13265130053 023004 0 ustar klemms o0cvpr import functools
from PyQt5 import QtCore, QtWidgets
from PyQt5.QtCore import Qt
from PyQt5.QtWidgets import QMenu, QAction, QMessageBox
import backend.caffe.dict_helper as helper
import gui.main_window.docks.properties.property_widgets.data as PropData
from gui.main_window.docks.dock import DockElement
class DockElementActivLayers(DockElement):
def __init__(self, mainWindow, title):
DockElement.__init__(self, mainWindow, 'Network Layers')
self.name = title
# Set the minimum size parameters
self.setMinimumHeight(265)
self.setMinimumWidth(250)
widget = QtWidgets.QWidget()
self.layout = QtWidgets.QVBoxLayout()
widget.setLayout(self.layout)
self.setWidget(widget)
self.addTextfieldForSearch()
self.addTableWidget()
def addTextfieldForSearch(self):
textField = QtWidgets.QLineEdit(self)
textField.setPlaceholderText("Search for layers")
textField.textChanged.connect(self.searchForLayers)
self.layout.addWidget(textField)
def addTableWidget(self):
self.table = activLayerTable()
self.layout.addWidget(self.table)
def searchForLayers(self, layerName):
self.table.searchForLayers(layerName)
def getLayersListWidget(self):
return self.table
def mousePressEvent(self, QMouseEvent):
return
def disableEditing(self, disable):
''' Disable the table with all active layers. '''
if self.table:
self.table.setDisabled(disable)
class activLayerTable(QtWidgets.QTableWidget):
def __init__(self):
QtWidgets.QTableWidget.__init__(self)
# Set behaviour and layout
self.setHorizontalHeader(HorizontalHeader(QtCore.Qt.Horizontal, self))
self.horizontalHeader().setTableWidget(self)
self.horizontalHeader().setSectionResizeMode(QtWidgets.QHeaderView.ResizeToContents)
self.horizontalHeader().setStretchLastSection(True)
self.setSelectionMode(QtWidgets.QAbstractItemView.SingleSelection)
self.setSelectionBehavior(QtWidgets.QAbstractItemView.SelectRows)
# Set the number of columns as well as their headers, also disabled the vertical headers
self.setColumnCount(3)
self.verticalHeader().setVisible(False)
self.setHorizontalHeaderItem(0, QtWidgets.QTableWidgetItem("Order"))
self.setHorizontalHeaderItem(1, QtWidgets.QTableWidgetItem("Name"))
self.setHorizontalHeaderItem(2, QtWidgets.QTableWidgetItem("Type"))
# Add signal handling when cell is changed by user input
self.itemChanged.connect(self.cellIsChanged)
# Initialize internal variables
self.tableItems = []
self.tableItemsSearch = []
self.selectedItemRows = []
self.selectedItemIds = []
# Store if the selection was changed by the keyboard or some other event
self.selectionByKeyboard = 0
# Set up a variable to store the sorting order, default being Ascending by Name
self.sortingOrder = (0, QtCore.Qt.AscendingOrder)
self.reactiveState = None
def addItem(self, id, name,order, type):
# Add a new item to the internal data list and update the ui accordingly
self.tableItems.append(TableItem(id, name,order, self.networkManager.getToolTip(id), type))
self.updateWithAllElements()
# Sort the items so that the new item fits in at the right place
self.sort()
def __onUpdate(self, uri, reason):
if uri[0] == "selection":
self.clearSelection()
for id in self.reactiveState["selection"]:
self.addItemToSelection(id)
if uri[0] == "network":
network = self.reactiveState["network"]
uri = uri[1:]
if uri == ["layerOrder"]:
for i, id in enumerate(network["layerOrder"]):
self.updateItemOrder(id, i)
self.updateWithAllElements()
return
if len(uri) > 1:
id = uri[1]
if uri[2:] == ["parameters","name"]:
if isinstance(reason, PropData.ReasonUpdated):
self.updateItemName(id, reason.newValue)
else:
if isinstance(reason, PropData.ReasonKeyAdded):
id = reason.key
if id in network["layerOrder"]:
order = network["layerOrder"].index(id)
else:
order = 0
name = network["layers"][id]["parameters"]["name"]
typename = network["layers"][id]["type"].name()
self.addItem(id, name, order, typename)
if isinstance(reason, PropData.ReasonKeyDeleted):
id = reason.key
self.deleteItemsByID([id])
def setReactiveState(self, state):
if self.reactiveState:
self.reactiveState.someChildPropertyChanged.disconnect(self.__onUpdate)
self.reactiveState = state
if self.reactiveState:
self.reactiveState.someChildPropertyChanged.connect(self.__onUpdate)
def setSelectedItem(self,id):
self.selectionByKeyboard -= 1
self.clearSelection()
for row in range(0, self.rowCount()):
if id == self.item(row, 1).id:
self.item(row, 0).setSelected(True)
self.item(row, 1).setSelected(True)
self.item(row, 2).setSelected(True)
# Set this item as the last one selected
self.selectedItemRows.append(row)
self.selectedItemIds.append(id)
# Make sure the item is visible by scrolling to it
self.scrollToItem(self.item(row,0))
break
def addItemToSelection(self, id):
self.selectionByKeyboard -= 1
for row in range(0, self.rowCount()):
if id == self.item(row, 1).id:
self.item(row, 0).setSelected(True)
self.item(row, 1).setSelected(True)
self.item(row, 2).setSelected(True)
# Set this item as the last one selected
self.selectedItemRows.append(row)
self.selectedItemIds.append(id)
# Make sure the item is visible by scrolling to it
self.scrollToItem(self.item(row, 0))
break
def removeItemFromSelection(self, id):
self.selectionByKeyboard -= 1
for row in range(0, self.rowCount()):
if id == self.item(row, 1).id:
self.item(row, 0).setSelected(False)
self.item(row, 1).setSelected(False)
self.item(row, 2).setSelected(False)
# Set this item as the last one selected
self.selectedItemRows.remove(row)
self.selectedItemIds.remove(id)
# Make sure the item is visible by scrolling to it
self.scrollToItem(self.item(row, 0))
break
def setSelectedItems(self, ids):
self.selectionByKeyboard -= 1
self.clearSelection()
for row in range(0, self.rowCount()):
for id in ids:
if id == self.item(row, 1).id:
self.item(row, 0).setSelected(True)
self.item(row, 1).setSelected(True)
self.item(row, 2).setSelected(True)
# Set this item as the last one selected
self.selectedItemRows.append(row)
self.selectedItemIds.append(id)
# If there is only one selected item, make it visible by scrolling to it
if len(ids) == 1:
self.scrollToItem(self.item(row, 0))
break
def clearSelection(self):
self.setCurrentItem(None)
self.selectedItemRows = []
self.selectedItemIds = []
def deleteItemsByID(self, ids):
for id in ids:
for i in range(0, len(self.tableItems)):
if self.tableItems[i].id == id:
del self.tableItems[i]
break;
# Update the list with all elements
self.updateWithAllElements()
def clearTable(self):
self.clearContents()
self.tableItems = []
self.tableItemsSearch = []
def updateWithAllElements(self):
self.update(self.tableItems)
def update(self, list):
# Only delete the contents so that the headers are still left
self.clearContents()
# Set the size of the widget
self.setRowCount(len(list))
# Set all items with their corresponding positions, then sort them
for row in range(0, len(list)):
self.setItem(row, 0, TableItemWidgetOrder(list[row].order, list[row].id, list[row].toolTip))
self.setItem(row, 1, TableItemWidgetName(list[row].name, list[row].id, list[row].toolTip))
self.setItem(row, 2, TableItemWidgetType(list[row].type, list[row].toolTip))
self.sort()
# Re-select the items that had been selected
for row in range(0, self.rowCount()):
for id in self.selectedItemIds:
if self.item(row, 1).id == id:
self.item(row, 0).setSelected(True)
self.item(row, 1).setSelected(True)
self.item(row, 2).setSelected(True)
break
def sort(self):
'Sorts the items according to the sorting option used by user, default is Ascending Order by Name'
self.sortItems(self.sortingOrder[0], self.sortingOrder[1])
def updateItemName(self, id, name):
for row in range(0, len(self.tableItems)):
if self.tableItems[row].getID() == id:
self.tableItems[row].setName(name)
break
self.updateWithAllElements()
def updateItemOrder(self, id, order):
for row in range(0,len(self.tableItems)):
if self.tableItems[row].getID() == id:
self.tableItems[row].setOrder(order)
break
def searchForLayers(self, layerName):
self.tableItemsSearch = []
for i in range(0, len(self.tableItems)):
if layerName.lower() in self.tableItems[i].name.lower():
self.tableItemsSearch.append(self.tableItems[i])
# Update the table with the searched items, so that only those are shown
self.update(self.tableItemsSearch)
def scrollToItemByID(self, id):
for row in range(0, self.rowCount() - 1):
if self.item(row, 0).getID() == id:
self.scrollToItem(self.item(row, 0))
def cellIsChanged(self, tableItem):
if isinstance(tableItem, TableItemWidgetName):
if(tableItem.isEdited()):
# Store the table item ID
id = tableItem.getID()
# Change the name of the item in the internal table and all other elements
#self.networkManager.updateName(tableItem.getID(), tableItem.text())
h = helper.DictHelper(self.networkManager.network) # type: helper.DictHelper
h.layerParams(id)["name"] = tableItem.text()
# Scroll to the renamed item to make sure that it is visible
self.scrollToItemByID(id)
def mousePressEvent(self, QMouseEvent):
# Check if the user has clicked an item
if self.itemAt(QMouseEvent.pos()) != None:
# Get the current row and id of the item belonging to it
row = self.itemAt(QMouseEvent.pos()).row()
id = self.item(row, 1).getID()
# On left button change the selection
if QMouseEvent.button() == Qt.LeftButton:
# Depending on whether the user is pressing the ctrl or shift key, add to or range selection
if QMouseEvent.modifiers() & Qt.ControlModifier:
# If the item is already selected, remove it from selection
if self.item(row, 1).isSelected():
self.networkManager.removeLayerFromSelection(id)
else:
self.networkManager.addLayerToSelection(id)
elif QMouseEvent.modifiers() & Qt.ShiftModifier:
# The Row of the last selected element is stored as the last element of selectedItemRows
lastItemRow = self.selectedItemRows[len(self.selectedItemRows) - 1]
# Clear the selection
# self.networkManager.clearSelection()
# Select items depending on the position of the clicked item
if lastItemRow < row:
# Start at row and select everything to the last selected element
for i in range(lastItemRow, row + 1):
self.networkManager.addLayerToSelection(self.item(i, 1).getID())
else:
# Start at last selected item and select everything up to row
for i in range(lastItemRow, row - 1, -1):
self.networkManager.addLayerToSelection(self.item(i, 1).getID())
else:
self.networkManager.setSelection(id)
# On right button open the context menu
if QMouseEvent.button() == Qt.RightButton:
# Set the selection to the clicked item
self.networkManager.setSelection(id)
# Show the context menu
self.showContextMenu(QMouseEvent.globalPos(), id)
else:
# Otherwise, clear the selection
self.networkManager.clearSelection()
def mouseMoveEvent(self, QMouseEvent):
return
def mouseReleaseEvent(self, QMouseEvent):
return
def mouseDoubleClickEvent(self, QMouseEvent):
# Check if user clicked an actual item
clickedItem = self.itemAt(QMouseEvent.pos())
# Enable editing of the item on double click, editing is only enabled for the name column
if isinstance(clickedItem, TableItemWidgetName):
clickedItem.setEdited(True)
self.editItem(self.itemAt(QMouseEvent.pos()))
def keyReleaseEvent(self, QKeyEvent):
# Delete on del key
if QKeyEvent.key() == QtCore.Qt.Key_Delete:
self.networkManager.deleteSelectedLayers()
# Switch selection to next item on tab
elif QKeyEvent.key() == QtCore.Qt.Key_Tab:
# Get the row of the last selected element
row = self.selectedItemRows[len(self.selectedItemRows) - 1]
# If the last item has been reached start at the beginning
if row == self.rowCount() - 1:
self.networkManager.setSelection(self.item(0, 1).id)
else:
self.networkManager.setSelection(self.item(row + 1, 1).id)
# Switch to or add to selection on arrow down
elif QKeyEvent.key() == QtCore.Qt.Key_Down:
if self.rowCount() == 0:
return # abort
# Get the row of the last selected element
if len(self.selectedItemRows) > 0:
row = max(self.selectedItemRows)
else:
row = -1
# If last selection event was not done via keyboard, clear the selection
# if self.selectionByKeyboard < 0:
# try:
# self.networkManager.setSelection(self.item(row, 1).id)
# except AttributeError:
# pass
# Save that the last edit was done by keyboard
self.selectionByKeyboard = 1
# If the last item has been reached do nothing
if not row == self.rowCount() - 1:
# Add to the selection if shift is held down
if QKeyEvent.modifiers() & Qt.ShiftModifier:
# If the next item is already selected, remove the current from selection
if self.selectedItemRows.__contains__(row + 1):
if self.selectedItemRows.__contains__(row):
self.networkManager.removeLayerFromSelection(self.item(row, 1).id)
else:
self.networkManager.addLayerToSelection(self.item(row + 1, 1).id)
else:
self.networkManager.setSelection(self.item(row + 1, 1).id)
self.currentSelectedRow = self.item(row + 1, 1).id
# Switch to or add to selection on arrow up
elif QKeyEvent.key() == QtCore.Qt.Key_Up:
if self.rowCount() == 0:
return # abort
# Get the row of the last selected element
# row = self.selectedItemRows[len(self.selectedItemRows) - 1]
if len(self.selectedItemRows) > 0:
row = min(self.selectedItemRows)
else:
row = self.rowCount()
# If last selection event was not done via keyboard, clear the selection
# if self.selectionByKeyboard < 0:
# try:
# self.networkManager.setSelection(self.item(row, 1).id)
# except AttributeError:
# pass
# Save that the last edit was done by keyboard
self.selectionByKeyboard = 1
# If the first item has been reached do nothing
if not row == 0:
# Add to the selection if shift is held down
if QKeyEvent.modifier() & Qt.ShiftModifier:
if self.selectedItemRows.__contains__(row - 1):
if self.selectedItemRows.__contains__(row):
self.networkManager.removeLayerFromSelection(self.item(row, 1).id)
else:
self.networkManager.addLayerToSelection(self.item(row - 1, 1).id)
else:
self.networkManager.setSelection(self.item(row - 1, 1).id)
def showContextMenu(self, pos, id):
# Create a menu with context options
menu = QMenu(self)
# Add a focus option to context menu
focusAction = QAction("Focus", self)
focusAction.triggered.connect(functools.partial(self.networkManager.focusLayer, id))
menu.addAction(focusAction)
# Add a delete option to context menu
deleteAction = QAction('Delete', self)
deleteAction.triggered.connect(functools.partial(self.networkManager.deleteLayer, id))
menu.addAction(deleteAction)
# Show the menu at the current mouse position
menu.popup(pos)
def setNetworkManager(self, networkManager):
self.networkManager = networkManager
def getNetworkManager(self):
return self.networkManager
def setSortingOrder(self, column, order):
self.sortingOrder = (column, order)
def getSortingOrder(self):
return self.sortingOrder
class TableItem():
def __init__(self, id, name,order, toolTip, type):
self.name = name
self.id = id
self.order = order
self.toolTip = toolTip
self.type = type
def getID(self):
return self.id
def getName(self):
return self.name
def setName(self,name):
self.name = name
def setOrder(self,order):
self.order = order
class TableItemWidgetOrder(QtWidgets.QTableWidgetItem):
def __init__(self, order,id, tooltip):
QtWidgets.QTableWidgetItem.__init__(self, self._longString(order))
self.id = id
self.order = order
#self.setToolTip(tooltip)
self.setFlags(QtCore.Qt.ItemIsSelectable | QtCore.Qt.ItemIsEnabled)
def getID(self):
return self.id
def setOrder(self, order):
self.order = order
def getOrder(self):
return self.order
def _longString(self,number):
strg = str(number)
while len(strg)<3:
strg = "0"+strg
return strg
class TableItemWidgetName(QtWidgets.QTableWidgetItem):
def __init__(self, name, id, tooltip):
QtWidgets.QTableWidgetItem.__init__(self, name)
self.name = name
self.id = id
self.edited = False
#self.setToolTip(tooltip)
def setEdited(self, bool):
self.edited = bool
def isEdited(self):
return self.edited
def getID(self):
return self.id
def setName(self, name):
self.name = name
def getName(self):
return self.name
class TableItemWidgetType(QtWidgets.QTableWidgetItem):
def __init__(self, type, tooltip):
QtWidgets.QTableWidgetItem.__init__(self, type)
self.setFlags(QtCore.Qt.ItemIsSelectable | QtCore.Qt.ItemIsEnabled)
#self.setToolTip(tooltip)
class HorizontalHeader(QtWidgets.QHeaderView):
def __init(self, orientation, parent):
QtWidgets.QHeaderView.__init__(orientation, parent)
def setTableWidget(self, tableWidget):
# Save the parent widget for access to current sorting order
self.tableWidget = tableWidget
# Set Sorting indicator so that it is Ascending Order by Name, then show sorting indicator
self.setSortIndicator(1, QtCore.Qt.AscendingOrder)
self.setSortIndicatorShown(True)
def mousePressEvent(self, QMouseEvent):
# Get the index from the element itself and the current order from the table widget
index = self.logicalIndexAt(QMouseEvent.pos())
currentOrder = self.tableWidget.getSortingOrder()[1]
# Set the new sorting order depending on the current order and clicked element
if currentOrder == QtCore.Qt.AscendingOrder:
self.tableWidget.setSortingOrder(index, QtCore.Qt.DescendingOrder)
self.tableWidget.sort()
self.setSortIndicator(index, QtCore.Qt.DescendingOrder)
if currentOrder == QtCore.Qt.DescendingOrder:
self.tableWidget.setSortingOrder(index, QtCore.Qt.AscendingOrder)
self.tableWidget.sort()
self.setSortIndicator(index, QtCore.Qt.AscendingOrder)
# Show the sorting indicator
self.setSortIndicatorShown(True)
def mouseMoveEvent(self, QMouseEvent):
return
def mouseReleaseEvent(self, QMouseEvent):
return
Barista/gui/main_window/docks/dock_available_layers.py 0000664 ڊ0011117 00000011522 13265130053 023440 0 ustar klemms o0cvpr from PyQt5 import QtCore, QtWidgets
from PyQt5 import QtGui
import backend.caffe.proto_info as info
from gui.main_window.docks.dock import DockElement
class DockElementLayers(DockElement):
def __init__(self, mainWindow, title):
# Set title and initialize the dock widget
DockElement.__init__(self, mainWindow, title)
self.name = title
widget = QtWidgets.QWidget()
self.layout = QtWidgets.QVBoxLayout()
# Add the text field and the layer list to the widget
self.addDescription()
self.addTextfieldForSearch()
self.addListOfAllLayers()
self.allLayersDict = info.CaffeMetaInformation().availableLayerTypes()
self.allLayers = self.dictionaryToList(self.allLayersDict)
self.setAllLayersNewLayerList(self.allLayers)
widget.setLayout(self.layout)
self.setWidget(widget)
def addDescription(self):
descriptionLabel = QtWidgets.QLabel("Drag and drop any of the available layers to the network canvas to add a new layer.", self)
descriptionLabel.setObjectName("infoText")
descriptionLabel.setWordWrap(True)
# Set the object name so that the global stylesheet for info text will be applied.
# descriptionLabel.setStyleSheet("color: rgba(0, 0, 0, 125)")
self.layout.addWidget(descriptionLabel)
def addTextfieldForSearch(self):
textField = QtWidgets.QLineEdit(self)
textField.setPlaceholderText("Search for Layers")
textField.textChanged.connect(self.searchForLayers)
self.layout.addWidget(textField)
def addListOfAllLayers(self):
#used: http://pyqt.sourceforge.net/Docs/PyQt4/qlistwidget.html
self.allLayersWindow = QtWidgets.QListWidget(self)
# Enable dragging of list elements, connect item pressed action so that current item is saves for drag and drop
self.allLayersWindow.setDragEnabled(True)
self.allLayersWindow.itemPressed.connect(self.startDragAction)
#show all Layers in list
self.layout.addWidget(self.allLayersWindow)
self.updateListOfAllLayers()
def updateListOfAllLayers(self):
info.CaffeMetaInformation().updateCaffeMetaInformation()
self.allLayersDict = info.CaffeMetaInformation().availableLayerTypes()
self.allLayers = self.dictionaryToList(self.allLayersDict)
self.setAllLayersNewLayerList(self.allLayers)
self.updateAllLayersWindowNewList(self.allLayers)
def dictionaryToList(self,dictionary):
list = [None]*len(dictionary)
for i in range(0,len(dictionary)):
list[i] = dictionary.keys()[i]
return list
def setAllLayersNewLayerList(self,list):
self.allLayers = list
def setAllLayersAddLayerList(self, list):
'''this function adds the layername that are in list to the list self.allLayers'''
for i in range(0,len(list)):
isIn =1
for j in range(0,len(self.allLayers)):
if list[i] == self.allLayers[j]:
isIn = 0
if(isIn):
self.allLayers.append(list[i])
def updateAllLayersWindowNewList(self,newList):
'''this function updates the list that shows the Layer with a given list,
in this main_window are only the names of the layers represented'''
self.allLayersWindow.clear()
for i in range(0,len(newList)):
self.allLayersWindow.addItem(newList[i])
self.allLayersWindow.sortItems(0)
def updateAllLayersWindowAddList(self,listToAdd):
'''this function updates the layerwindow and adds the given list to the layers that are allready shown,
in this main_window are only the name of the layers represented'''
for i in range(0,len(listToAdd)):
if len(self.allLayersWindow.findItems(listToAdd[i],QtCore.Qt.MatchExactly)) is 0:
self.allLayersWindow.addItem(listToAdd[i])
self.allLayersWindow.sortItems(0)
#todo: write a function to search for a special layer and show only that one
def searchForLayers(self,layerName):
list = []
#search in list of all Layers which are layerName
for i in range(0,len(self.allLayers)):
if layerName.lower() in self.allLayers[i].lower():
list.append(self.allLayers[i])
self.updateAllLayersWindowNewList(list)
#todo: create a function to drag and drop the layer into the main main_window to create a new network
def startDragAction(self):
mimeData = QtCore.QMimeData()
mimeData.setText(self.allLayersWindow.currentItem().text())
drag = QtGui.QDrag(self)
drag.setMimeData(mimeData)
drag.exec_()
def disableEditing(self, disable):
""" Disable the table with all available layers. """
if self.allLayersWindow:
self.allLayersWindow.setDisabled(disable)
Barista/gui/main_window/docks/dock_console.py 0000664 ڊ0011117 00000015565 13265130053 021616 0 ustar klemms o0cvpr # -*- coding: utf-8 -*-
import traceback
import PyQt5.QtGui as QtGui
from PyQt5.Qt import QTextCursor
from PyQt5.QtWidgets import QWidget, QTextEdit, QLabel, QComboBox, QVBoxLayout, QHBoxLayout
from backend.barista.utils.logger import Log, Logger
from gui.main_window.docks.dock import DockElement
class DockElementConsole(DockElement):
def __init__(self, mainWindow, title):
DockElement.__init__(self, mainWindow, 'Console')
self.name = title
self.resize(700, 500)
self.widget = QWidget()
Log.newLine.connect(self.appendLineToConsole)
Log.sigRefreshGui.connect(self.appendLinesToConsole)
Log.sigRefreshCallers.connect(self.fillCallerComboBox)
self.__callersCopy = []
self.__currentCallerFilter = -1
self.__currentTypeFilter = -1
self.__msgTypes = [Logger.ALL, Logger.ERROR, Logger.TEXT]
self.__logOutput = None
self.__callerComboBox = None
self.__scrollDown = True
self.__setupGui()
def clearLog(self):
'''
clears all input in the edittext field
'''
self.__logOutput.clear()
def appendLineToConsole(self, logline):
'''
adds a line of input the the console
with prefix if it is not filtered
'''
if(self.__shouldDisplayLogLine(logline)):
try:
self.__setupFont(logline.caller, logline.msgType)
sb = self.__logOutput.verticalScrollBar()
if sb.value() >= sb.maximum():
self.__scrollDown = True
else:
self.__scrollDown = False
# self.__writePrefix(logline.caller) not used because there is no append..
logText = logline.line
self.__logOutput.append(self.__getPrefix(logline.caller, logline.time) + logText)
except:
errMsg = traceback.format_exc()
self.__logOutput.append("Error during printing a log line: {}".format(
errMsg
))
def __writePrefix(self, caller):
'''
NOT USED because of no appen of plain text
and self.__setCursorToEnd() leads to lag of gui
writes the prefix to the log
without line feed
'''
font = self.__logOutput.font()
font.setBold(True)
self.__logOutput.setCurrentFont(font)
self.__logOutput.insertPlainText(self.__getPrefix(caller))
font = self.__logOutput.font()
font.setBold(False)
self.__logOutput.setCurrentFont(font)
def __getPrefix(self, caller, time):
'''
the prefix that have to be added
'''
prefix = ""
if self.__currentCallerFilter == -1:
prefix = "[" + time + ", " + caller.description + "]"
return prefix
def __shouldDisplayLogLine(self, logline):
'''
returns whether this line should be displayed. Filtered on Caller
and MessageType
'''
if(self.__currentCallerFilter == -1):
if(self.__currentTypeFilter != -1):
return (logline.msgType.typeId == self.__currentTypeFilter)
return True
elif(logline.caller.callerId == self.__callersCopy[self.__currentCallerFilter].callerId):
if(self.__currentTypeFilter == -1):
return True
else:
return logline.msgType.typeId == self.__currentTypeFilter
return False
def appendLinesToConsole(self, loglines):
'''
adds multiple lines to the console
'''
for logline in loglines:
self.appendLineToConsole(logline)
def __setupFont(self, caller, msgType):
'''
setup the font
'''
font = self.__logOutput.font()
font.setBold(False)
color = caller.color
if(msgType == Logger.ERROR):
color = msgType.color
font.setFamily("Courier")
font.setPointSize(10)
self.__logOutput.setCurrentFont(font)
self.__logOutput.setTextColor(color)
def __setupGui(self):
'''
adds the wigets to the gui
'''
self.__logOutput = QTextEdit(self.widget)
self.__logOutput.setReadOnly(True)
self.__logOutput.setLineWrapMode(QTextEdit.NoWrap)
self.__logOutput.verticalScrollBar().rangeChanged.connect(self.handleSroll)
labelCallers = QLabel("Show output generated by ")
self.__callerComboBox = QComboBox()
self.__callerComboBox.currentIndexChanged.connect(self.__filterCallers)
labelTypes = QLabel("Show messages of type ")
self.__typeComboBox = QComboBox()
self.__typeComboBox.currentIndexChanged.connect(self.__filterTypes)
for type in self.__msgTypes:
self.__typeComboBox.addItem(type.description)
Log.refreshCallers()
vbox = QVBoxLayout()
hbox = QHBoxLayout()
hbox.addWidget(labelCallers)
hbox.addWidget(self.__callerComboBox)
hbox.addWidget(labelTypes)
hbox.addWidget(self.__typeComboBox)
hbox.addStretch(1)
vbox.addLayout(hbox)
vbox.addWidget(self.__logOutput)
self.widget.setLayout(vbox)
self.widget.resize(700, 500)
self.setWidget(self.widget)
def __setCursorToEnd(self):
'''
used to not insert new lines at the users
selection
'''
self.__logOutput.moveCursor(QTextCursor.End)
def fillCallerComboBox(self, callers):
'''
inserts the data for the filter caller combo box
callers is a dictionary callerid (int) -> Caller object
'''
#Display only callers that have content logged
filterCallers = lambda caller: caller.used == True
#save copy to get right index
self.__callersCopy = list(filter(filterCallers, callers.values()))
self.__callerComboBox.clear()
model = self.__callerComboBox.model()
# Appen All
item = QtGui.QStandardItem(str(Logger.ALL.description)) # no Error
model.appendRow(item)
# AppendCallers
for caller in self.__callersCopy :
item = QtGui.QStandardItem(str(caller.description))
item.setForeground(caller.color)
model.appendRow(item)
def handleSroll(self):
'''
cares about whether the scrollbar has to be on bottom of
the console or not
'''
sb = self.__logOutput.verticalScrollBar()
if(self.__scrollDown):
sb.setValue(sb.maximum())
def __filterCallers(self, i):
if(i == -1):
return; # invalid filter, empty box or resetet
self.__currentCallerFilter = i-1 # -1 Because of "ALL"
self.clearLog()
Log.refreshConsole()
def __filterTypes(self, i):
self.__currentTypeFilter = i-1 # -1 Because of "ALL"
self.clearLog()
Log.refreshConsole()
Barista/gui/main_window/node_editor/__init__.py 0000664 ڊ0011117 00000000000 13265130053 022055 0 ustar klemms o0cvpr Barista/gui/main_window/node_editor/layer_color_definitions.py 0000664 ڊ0011117 00000004151 13265130053 025236 0 ustar klemms o0cvpr from PyQt5.QtGui import QColor
from backend.caffe.proto_info import LayerType
from gui.main_window.node_editor.node_editor_constants import Constants
class LayerColorDefinitions:
__LAYER_CATEGORY_COLORS = {LayerType.CATEGORY_DATA : QColor( 0, 150, 0),
LayerType.CATEGORY_LOSS : QColor( 0, 100, 255),
LayerType.CATEGORY_ACTIVATION : QColor( 200, 200, 50),
LayerType.CATEGORY_NONE : Constants.itemBackgroundColorLight}
__LAYER_GROUP_INVALID = -1
__LAYER_GROUP_PYTHON_LAYER = 0
__LAYER_GROUP_STRUCTURAL_LAYER = 1
__LAYER_GROUP_COLORS = {__LAYER_GROUP_PYTHON_LAYER : QColor(50, 255, 255),
__LAYER_GROUP_STRUCTURAL_LAYER : QColor(255, 100, 0)}
__LAYER_GROUP_DEFINITIONS = {"Python" : __LAYER_GROUP_PYTHON_LAYER,
"Convolution" : __LAYER_GROUP_STRUCTURAL_LAYER,
"InnerProduct": __LAYER_GROUP_STRUCTURAL_LAYER,
"Deconvolution": __LAYER_GROUP_STRUCTURAL_LAYER,
"LSTM": __LAYER_GROUP_STRUCTURAL_LAYER,
"RNN": __LAYER_GROUP_STRUCTURAL_LAYER,
"BatchNorm": __LAYER_GROUP_STRUCTURAL_LAYER}
def __init__(self):
return
@staticmethod
def getTypeColor(nodeType):
""" Returns the color for a given layer type based on the types category and type name """
# First try to get the group by type name. If the type name is found, try to get the groups color.
# If the type name has an invalid group, the layers category is used to get the color.
# If the type name can not be found, use the layer category as well.
return LayerColorDefinitions.__LAYER_GROUP_COLORS.get(
LayerColorDefinitions.__LAYER_GROUP_DEFINITIONS.get(nodeType.name(),
LayerColorDefinitions.__LAYER_GROUP_INVALID),
LayerColorDefinitions.__LAYER_CATEGORY_COLORS[nodeType.category()])
Barista/gui/main_window/node_editor/node_editor.py 0000664 ڊ0011117 00000066722 13265130053 022640 0 ustar klemms o0cvpr from PyQt5.QtCore import QPointF
from PyQt5.QtWidgets import QInputDialog, QMessageBox
from gui.main_window.node_editor.items.connection_item import ConnectionItem
from gui.main_window.node_editor.items.node_item import NodeItem
from gui.main_window.node_editor.node_editor_event_handler import NodeEditorEventHandler
from gui.main_window.node_editor.node_editor_view import NodeEditorView
from gui.main_window.node_editor.node_sort import NodeSort
class NodeEditor:
def __init__(self, parent=None):
self.__view = NodeEditorView(self, parent)
self.__scene = self.__view.scene()
self.__networkManager = None
self.__currentPhase = ""
self.__nodes = dict()
self.__hiddenConnections = list()
self.__eventHandler = NodeEditorEventHandler(self, self.__view, self.__scene)
self.__view.setEventHandler(self.__eventHandler)
self.__scene.setEventHandler(self.__eventHandler)
self.disable = False
def getNodes(self):
""" Returns the dict of all node items, the layer ID is the key """
return self.__nodes
def setNetworkManager(self, networkManager):
""" Sets the NetworkManager """
self.__networkManager = networkManager
self.__eventHandler.setNetworkManager(networkManager)
def getNetworkManager(self):
""" Returns the NetworkManager """
return self.__networkManager
def setCopySelection(self):
self.__eventHandler.setCopySelection()
def clearAll(self):
""" Removes all items from the scene and clears the internal dicts """
self.__nodes.clear()
self.__scene.clear()
def createLayerPositionDict(self):
""" Creates and returns a dict with the layer ID as key and the layers position in the scene as value """
# create a new dict
positions = dict()
# fill dict
for layerID, nodeItem in self.__nodes.iteritems():
pos = nodeItem.scenePos()
positions[layerID] = (pos.x(), pos.y())
return positions
def getPositionOfLayer(self, id):
""" Returns the position of a node item with a given layer ID """
pos = self.__nodes[id].scenePos()
return pos.x(), pos.y()
def setPositionOfLayer(self, id, pos):
""" Sets the position of a node item with the given layer ID """
self.__nodes[id].setPos(pos[0],pos[1])
def applyLayerPositionDict(self, positionsDict):
""" Sets the positions of all node items to the positions defined in positionsDict, layerID is the key """
for layerID, pos in positionsDict.iteritems():
self.__nodes[layerID].setPos(pos[0], pos[1])
def addLayer(self, layerID, layerData, scenePosX = 0.0, scenePosY = 0.0):
""" Creates a new NodeItem to represent the layer """
# create the new node Item
nodeItem = NodeItem(layerID, layerData, self)
# set the tooltip containing the parameters to the node item
parameters = self.__networkManager.getToolTip(layerID)
nodeItem.setToolTip(parameters)
# add top connectors to the node item
if "top" in layerData["parameters"]:
for top in layerData["parameters"]["top"]:
nodeItem.addTopConnector(top)
# add bottom connectors to the node item
if "bottom" in layerData["parameters"]:
for bottom in layerData["parameters"]["bottom"]:
nodeItem.addBottomConnector(bottom)
# add the node item to the scene and the internal dict
self.__scene.addItem(nodeItem)
self.__nodes[layerID] = nodeItem
nodeItem.setPos(QPointF(scenePosX, scenePosY))
def setKeyboardFocus(self):
self.__view.setFocus()
def focusLayer(self, layerID):
""" Focuses the viewport to the node item with the given layer ID """
position = self.__nodes[layerID].pos()
self.__view.centerOn(position)
def addBottomBlob(self, layerID, blobName):
""" Adds a bottom connector to the node item with the given layer ID """
nodeItem = self.__nodes[layerID]
nodeItem.addBottomConnector(blobName)
def removeBottomBlob(self, layerID, blobIndex):
""" Removes a bottom connector from the node item with the given layer ID """
self.__nodes[layerID].removeBottomConnector(blobIndex)
# Update the tooltip as well
self.updateTooltip(layerID)
def addTopBlob(self, layerID, blobName):
""" Adds a top connector to the node item with the given layer ID """
nodeItem = self.__nodes[layerID]
nodeItem.addTopConnector(blobName)
bottomConnectors = self.__nodes[layerID].getBottomConnectors()
for bottomConnector in bottomConnectors:
connections = bottomConnector.getConnections()
for connection in connections:
connection.updateData()
# Update the tooltip as well
self.updateTooltip(layerID)
def renameTopBlob(self, layerID, blobIndex, newName):
""" Renames a top connector and changes the name of all connected bottom connectors """
# update the top connectors name
connector = self.__nodes[layerID].getTopConnectors()[blobIndex]
connector.setBlobName(newName)
self.__nodes[layerID].updateLayerData()
# get all connected connectors and change their name
connections = connector.getConnections()
for connection in connections:
bottomConnector = connection.getBottomConnector()
self.__networkManager.renameBottomBlob(bottomConnector.getNodeItem().getLayerID(),
bottomConnector.getIndex(),
newName)
connection.updateData()
# update all connections
bottomConnectors = self.__nodes[layerID].getBottomConnectors()
for bottomConnector in bottomConnectors:
connections = bottomConnector.getConnections()
for connection in connections:
connection.updateData()
# Update the tooltip as well
self.updateTooltip(layerID)
def renameBottomBlob(self, layerID, blobIndex, newName):
""" Renames a bottom connector and removes all connections getting invalid by this operation """
# rename the bottom connector
bottomConnector = self.__nodes[layerID].getBottomConnectors()[blobIndex]
bottomConnector.setBlobName(newName)
self.__nodes[layerID].updateLayerData()
# get all top connectors and check if they have the same name. If not -> remove connection
connections = list(bottomConnector.getConnections())
for connection in connections:
topConnector = connection.getTopConnector()
if topConnector.getBlobName() != newName:
self.removeConnection(connection)
# Update the tooltip as well
self.updateTooltip(layerID)
def removeTopBlob(self, layerID, blobIndex):
""" Removes a top connector from the node item with the given layer ID """
self.__nodes[layerID].removeTopConnector(blobIndex)
# Update the tooltip as well
self.updateTooltip(layerID)
def disconnectBottomBlob(self, layerID, blobIndex):
""" Removes all connections connected to the bottom connector with the given index
of the node item with the given layer ID """
connector = self.__nodes[layerID].getBottomConnectors()[blobIndex]
self.__disconnectAllConnectorConnections(connector)
# Update the tooltip as well
self.updateTooltip(layerID)
def disconnectTopBlob(self, layerID, blobIndex):
""" Removes all connections connected to the top connector with the given index
of the node item with the given layer ID """
connector = self.__nodes[layerID].getTopConnectors()[blobIndex]
self.__disconnectAllConnectorConnections(connector)
# Update the tooltip as well
self.updateTooltip(layerID)
def createConnection(self, topLayerID, topBlobIndex, bottomLayerID, bottomBlobIndex):
""" Creates a new connection between the top connector at topBlobIndex of the node item with the topLayerID
and the bottom connector at bottomBlobIndex of the node item with the bottomLayerID """
# get the connectors
topConnector = self.__nodes[topLayerID].getTopConnectors()[topBlobIndex]
bottomConnector = self.__nodes[bottomLayerID].getBottomConnectors()[bottomBlobIndex]
# create a new connection item and add it to the scene
connection = ConnectionItem(self)
self.__scene.addItem(connection)
# set the connection/connectors in the connection/connectors
connection.setTopConnector(topConnector)
connection.setBottomConnector(bottomConnector)
bottomConnector.addConnection(connection)
topConnector.addConnection(connection)
def rearrangeNodes(self):
""" Automatically sorts the node items """
nodesort = NodeSort(self.getNodes(), self.__view)
self.__networkManager.checkLayersForMovement(self.getNodeIds())
def rearrangeNodesVertical(self):
""" Automatically sorts the node items Vertical """
nodesort = NodeSort(self.getNodes(), self.__view, True)
self.__networkManager.checkLayersForMovement(self.getNodeIds())
def clearSelection(self):
""" Clears the selection """
self.__scene.clearSelection()
def setSelectedLayer(self, layerID):
""" Sets a single layer as selected. Already selected layers get deselected. """
# remove all node items from the selection
self.clearSelection()
# get the node item with layer ID and set it as selected
for item in self.__scene.items():
if isinstance(item, NodeItem):
if item.getLayerID() == layerID:
item.setSelected(True)
break
def setSelectedLayers(self, layerIDs):
""" Sets a list of layers as selected. Alread selected layers get deselected. """
# remove all node items from the selection
self.clearSelection()
# get all items in the scene
layerList = list(layerIDs)
items = self.__scene.items()
# check all items and set the as selected if they are in the given list
while(len(layerList) > 0):
for item in items:
if isinstance(item, NodeItem):
if item.getLayerID() == layerList[0]:
item.setSelected(True)
break
del layerList[0]
def addLayerToSelection(self, layerID):
""" Selects a node item without clearing other layers selection """
items = self.__scene.items()
for item in items:
if isinstance(item, NodeItem):
if item.getLayerID() == layerID:
item.setSelected(True)
break
def removeLayerFromSelection(self, layerID):
""" Deselects a node item without clearing other layers selection """
selectedItems = self.__scene.selectedItems()
for item in selectedItems:
if isinstance(item, NodeItem):
if item.getLayerID() == layerID:
item.setSelected(False)
break
def updateLayerData(self, layerID):
""" Notifies a node item to update the layer data to render correctly after change """
self.__nodes[layerID].updateLayerData()
# Update the tooltip as well
self.updateTooltip(layerID)
def disconnectConnectionsOfSelection(self):
""" Removes all connections from the selection and all connections connecting any selected node item """
# get all selected items
selectedItems = self.__scene.selectedItems()
# remove all selected connections
for item in selectedItems:
if isinstance(item, ConnectionItem):
self.removeConnection(item)
# remove all connections connected to selected layers
for item in selectedItems:
if isinstance(item, NodeItem):
self.__disconnectAllLayerConnections(item)
def __disconnectAllLayerConnections(self, nodeItem):
""" Removes all connections connected to any top or bottom connector of a given node item """
# remove connections of top connectors
for topConnector in nodeItem.getTopConnectors():
self.__disconnectAllConnectorConnections(topConnector)
# remove connections of bottom connectors
for bottomConnector in nodeItem.getBottomConnectors():
self.__disconnectAllConnectorConnections(bottomConnector)
def __disconnectAllConnectorConnections(self, connector):
""" Removes all connections connected to a connector """
connections = list(connector.getConnections())
for connection in connections:
self.removeConnection(connection)
def removeConnection(self, connection):
""" Removes a connection """
# get all selected items
selectedItems = self.__scene.selectedItems()
# remove connection from connectors
topConnector = connection.getTopConnector()
bottomConnector = connection.getBottomConnector()
topConnector.removeConnection(connection)
bottomConnector.removeConnection(connection)
# get layer ID and connector index to notify data change
bottomLayerID = bottomConnector.getNodeItem().getLayerID()
bottomBlobIndex = bottomConnector.getIndex()
# if the bottom connector has no connections left and is not in the selection,
# remove the underling data relation of top-bottom blob
if bottomConnector.getConnectionCount() == 0 and bottomConnector.getNodeItem() not in selectedItems:
self.__networkManager.disconnectLayer(bottomLayerID, bottomBlobIndex)
# remove the connection item from the scene
self.__scene.removeItem(connection)
def deleteItemsByID(self, list):
""" Removes node items from the scene based on the list of layer IDs """
for layerID in list:
items = self.__scene.items()
for item in items:
if isinstance(item, NodeItem):
if item.getLayerID() == layerID:
self.__scene.removeItem(item)
index = self.__nodes.values().index(item)
key = self.__nodes.keys()[index]
del self.__nodes[key]
def updateLayerName(self, layerID, name):
""" Notify the node item to update the rendering. """
self.__nodes[layerID].updateLayerData()
# Update the tooltip as well
self.updateTooltip(layerID)
def updateTooltip(self, layerID):
""" Updates the tooltip of the node item with the given layer ID """
parameters = self.__networkManager.getToolTip(layerID)
self.__nodes[layerID].setToolTip(parameters)
def createListFromHiddenConnections(self):
""" Creates a list of tuples containing the hidden connections, the tuples have the form
(topLayerID, topBlobIndex, bottomLayerID, bottomBlobIndex) """
outList = list()
for connection in self.__hiddenConnections:
topConnector = connection.getTopConnector()
bottomConnector = connection.getBottomConnector()
outList.append((topConnector.getNodeItem().getLayerID(), topConnector.getIndex(),
bottomConnector.getNodeItem().getLayerID(), bottomConnector.getIndex()))
return outList
def setHiddenConnectionsFromList(self, hiddenList):
""" Sets all connections to show and then hides the connections specified in hiddenList,
hiddenList contains tuples (topLayerID, topBlobIndex, bottomLayerID, bottomBlobIndex) """
self.__hiddenConnections = []
for data in hiddenList:
connection = self.__getConnection(data[0], data[1], data[2], data[3])
if connection is not None:
connection.setHidden(True)
self.__hiddenConnections.append(connection)
def tryToAddHiddenConnection(self, connection):
""" Called by connections to notify a hidden state change. """
if connection not in self.__hiddenConnections:
self.__hiddenConnections.append(connection)
self.__networkManager.connectionsHiddenStateChanged(self.createListFromHiddenConnections())
def tryToRemoveHiddenConnection(self, connection):
""" Called by connections to notify a hidden state change. """
if connection in self.__hiddenConnections:
self.__hiddenConnections.remove(connection)
self.__networkManager.connectionsHiddenStateChanged(self.createListFromHiddenConnections())
def tryToCreateLayer(self, type, scenePosX, scenePosY):
""" Notifies the NetworkManager to create a new Layer. """
self.__networkManager.addLayer(type, scenePosX, scenePosY)
def tryToAddBottomBlob(self, layerID, blobName):
""" Notifies the NetworkManager to create a new bottom connector for the given layer. """
self.__networkManager.addBottomBlob(layerID, blobName)
def tryToRemoveBottomBlob(self, layerID, blobIndex):
""" Notifies the NetworkManager to try to remove the bottom connector
with the given index in the given layer. """
self.__networkManager.removeBottomBlob(layerID, blobIndex)
def getNode(self, layerID):
return self.__nodes[layerID]
def tryToAddTopBlob(self, layerID, blobName):
""" Notifies the NetworkManager to add a top connector to the layer with the given name. """
self.__networkManager.addTopBlob(layerID, blobName)
def tryToRenameTopBlob(self, connector):
""" Shows a input dialog to ask for a new blob name
and notifies the NetworkManager to change the blobs name. """
# show input dialog
name, ok = QInputDialog.getText(self.__view, "Change name of top blob", "Top blob name:")
name = name.strip()
# if the name is invalid ask again
duplicateFound = True
while ok:
if len(name) == 0:
QMessageBox.warning(self.__view, "Top blob name is empty",
"The name for the top blob can't be empty or all whitespace characters.")
name, ok = QInputDialog.getText(self.__view, "Change name of top blob", "Top blob name:")
name = name.strip()
continue
elif duplicateFound:
for blob in connector.getNodeItem().getTopConnectors():
if name == blob.getBlobName() and blob != connector:
QMessageBox.warning(self.__view, "Top blob name already exists",
"The name you chose already exists on this Layer, please choose another name.")
name, ok = QInputDialog.getText(self.__view, "Change name of top blob", "Top blob name:")
name = name.strip()
duplicateFound = True
break # stop for loop, continue while loop
else:
duplicateFound = False
else:
break # if string is valid
# the user provided a valid name
if ok:
# check if any bottom connector has multiple connections as a bottom connector can only have one name
bottomConnectorWithMultipleConnectionsFound = False
connections = connector.getConnections()
for connection in connections:
bottomConnector = connection.getBottomConnector()
if bottomConnector.getConnectionCount() > 1:
bottomConnectorWithMultipleConnectionsFound = True
# ask the user whether they want to continue (break invalid connections) ore cancel
shouldRename = True
if bottomConnectorWithMultipleConnectionsFound:
reply = QMessageBox.question(self.__view, "Removing connections",
"A bottom blob, connected to the top blob, has multiple connections. "
"By renaming the top blob, the other connections will get removed. "
"Continue?", QMessageBox.Yes, QMessageBox.No)
if reply != QMessageBox.Yes:
shouldRename = False
# if the user did not cancel -> notify the Networkmanager
if shouldRename:
self.__networkManager.renameTopBlob(connector.getNodeItem().getLayerID(), connector.getIndex(), name)
def tryToRemoveTopBlob(self, layerID, blobIndex):
""" Notifies the NetworkManager to remove the bottom connector with the given index of the given layer """
self.__networkManager.removeTopBlob(layerID, blobIndex)
def tryToClearSelection(self):
""" Notifies the NetworkManager to clear the selection """
self.__networkManager.clearSelectionWithoutSavingHistory()
def tryToSetSelectionList(self, layerIDs):
""" Notifies the NetworkManager to update the selected layer list """
self.__networkManager.setSelectionList(layerIDs)
def tryToDeleteSelection(self):
""" Notifies the NetworkManager to remove all selected layers """
self.__networkManager.deleteSelectedLayers()
def tryToConnect(self, topLayerID, topBlobIndex, bottomLayerID, bottomBlobIndex):
""" Checks wheter the new connection would be valid and
notifies the NetworkManager to create the new connection """
topConnector = self.__nodes[topLayerID].getTopConnectors()[topBlobIndex]
bottomConnector = self.__nodes[bottomLayerID].getBottomConnectors()[bottomBlobIndex]
alreadyConnectedBottomLayers = topConnector.getConnectedLayers()
# check id the new connections is between two connectors of the same layer7
if topConnector.getNodeItem() == bottomConnector.getNodeItem():
return False
# check if connection between both connectors exists
if bottomLayerID in alreadyConnectedBottomLayers:
return False
# check if both layers have the same phase (if both are not in all phases)
if topConnector.getPhase() != "" and bottomConnector.getPhase() != "":
if topConnector.getPhase() != bottomConnector.getPhase():
return False
# if the top layer is in all phases, the bottom layer is not allowed to have any connection at the connector
if topConnector.getPhase() == "":
if bottomConnector.getConnectionCount() > 0:
return False
# if the top layer is not in all phases, check if the
# bottom layer connector has another connection in the same phase
else:
if bottomConnector.hasPhaseConnection(topConnector.getPhase()):
return False
elif bottomConnector.getBlobName() != "":
if topConnector.getBlobName() != bottomConnector.getBlobName():
return False
if topConnector.isInPlace():
if topConnector.hasPhaseConnection(bottomConnector.getPhase()):
return False
# layers can get connected
self.__networkManager.connectLayers(topLayerID, topBlobIndex, bottomLayerID, bottomBlobIndex)
return True
def isCurrentPhase(self, phase):
""" Returns whether the phase is equals the current displayed phase. """
if len(phase) == 0 or self.__currentPhase == phase:
return True
else:
return False
def __getConnection(self, topID, topIndex, bottomID, bottomIndex):
""" Returns the connection between the layers with the IDs topID and bottomID,
connected at the connectors with the indices topIndex and bottomIndex """
topConnector = self.__nodes[topID].getTopConnectors()[topIndex]
connections = topConnector.getConnections()
for connection in connections:
bottomConnector = connection.getBottomConnector()
if bottomConnector.getNodeItem().getLayerID() == bottomID and bottomConnector.getIndex() == bottomIndex:
return connection
return None
def calculateLayerOrder(self):
""" Returns a list of layer ID representing the order of connection """
order = list()
# get a list of all node items with no bottom connectors
# or with no connections connected to any bottom connector
currentNodes = self.__getStartNodes()
waitingNodes = dict()
# unused
touched = {}
# repeat until all nodes have been processed
while len(currentNodes) > 0:
currentNode = currentNodes[0]
touched[currentNode.getLayerID()] = currentNode.getName()
# add the current node to the order list
order.append(currentNode.getLayerID())
# get a list of all nodes connected to any top connector of the current node
followingNodes = currentNode.getNodesConnectedToTops()
for node in followingNodes:
touched[node.getLayerID()] = node.getName()
# for all following nodes, check if they are waiting (not all node items connected to bottoms
# have been processed) and decrees the waiting count. If all needed nodes have benn processed,
# add the node to the processable nodes
for following in followingNodes:
# the layer has already been processed
if following.getLayerID() in order:
continue
# layer has other unprocessed pre nodes
if following in waitingNodes:
if waitingNodes[following] > 1:
waitingNodes[following] -= 1
else:
currentNodes.append(following)
del waitingNodes[following]
else:
inputNodeCount = len(following.getNodesConnectedToBottoms())
if inputNodeCount > 1:
waitingNodes[following] = inputNodeCount - 1
else:
currentNodes.append(following)
currentNodes.remove(currentNode)
return order
def getView(self):
return self.__view
def getScene(self):
return self.__view.scene()
def __getStartNodes(self):
""" Returns a list of all node items with no bottom connectors
or no connections connected to bottom connectors. """
startNodes = list()
for layerID, item in self.__nodes.iteritems():
if item.getBottomConnectorCount() == 0:
startNodes.append(item)
else:
connected = False
for connector in item.getBottomConnectors():
if connector.getConnectionCount() > 0:
connected = True
break
if not connected:
startNodes.append(item)
return startNodes
def getNodeIds(self):
""" Returns a list of all layer IDs of nodes currently in the scene. """
return self.__nodes.keys()
def disableEditing(self, disable):
""" Disable relevant changes of the net. """
self.disable = disable
self.__eventHandler.setDisabled(disable)
self.__view.scene().disableEditing(disable)
for _, node in self.getNodes().iteritems():
node.disableBlobs(disable)
Barista/gui/main_window/node_editor/node_editor_constants.py 0000664 ڊ0011117 00000004715 13265130053 024726 0 ustar klemms o0cvpr from PyQt5.QtGui import QFont, QFontMetricsF, QColor
class Constants:
""" This class stores constant values used by the node prototxt_editor """
# grid size and pattern
sceneGridSize = 20
sceneLargeGridCells = 5
# scene size
sceneInitialWidth = 32000
sceneInitialHeight = 32000
# used for zoom in/out using the mouse wheel
viewScaleFactor = 1.1
viewMaxScale = 0.2
viewMinScale = 5.0
# use default font
font = QFont()
# font used for the layer name
nodeItemFontName = QFont(font)
nodeItemFontName.setPointSize(10)
nodeItemFontName.setBold(True)
nodeItemFontNameMetrics = QFontMetricsF(nodeItemFontName)
# font used for the layer type
nodeItemFontType = QFont(font)
nodeItemFontType.setPointSize(8)
nodeItemFontTypeMetrics = QFontMetricsF(nodeItemFontType)
# font used for the blob names
nodeItemFontBlob = QFont(font)
nodeItemFontBlob.setPointSize(8)
nodeItemFontBlobMetrics = QFontMetricsF(nodeItemFontBlob)
# constants used to add free spaces in the node item rendering
nodeItemMinBlobAreaHeight = 10
nodeItemHeadPadding = 20
nodeItemSelectionSize = 3
nodeItemBorderSize = 3
nodeItemTextMargin = 4
nodeItemConnectorPaddding = 6
# connector size
connectorItemOuterSize = 12
connectorItemInnerSize = 6
# opacity for items in phase and normal items
itemOpacityInPhase = 200
itemOpacityNotInPhase = 100
# node item background colors
itemBackgroundColorDark = QColor(100, 100, 100)
itemBackgroundColorLight = QColor(200, 200, 200)
itemBorderColor = QColor(0, 0, 0)
# color of items working in-place
itemInPlaceColor = QColor(30, 30, 200)
# connection item constants
connectionItemColor = QColor(100, 100, 100)
connectionItemSelectionSize = 9
connectionItemSize = 5
connectionItemHiddenDashCount = 4
connectionItemHiddenDashSize = 10
connectionItemHiddenDashSpace = 10
# color for selected items (node item, connection)
selectedColor = QColor(150, 150, 0)
def __init__(self):
return
@staticmethod
def setPaintDevice(device):
""" Used to setup font metrics correctly """
Constants.nodeItemFontNameMetrics = QFontMetricsF(Constants.nodeItemFontName, device)
Constants.nodeItemFontTypeMetrics = QFontMetricsF(Constants.nodeItemFontType, device)
Constants.nodeItemFontBlobMetrics = QFontMetricsF(Constants.nodeItemFontBlob, device)
Barista/gui/main_window/node_editor/node_editor_event_handler.py 0000664 ڊ0011117 00000074341 13265130053 025532 0 ustar klemms o0cvpr from PyQt5.QtCore import Qt, QPointF, QPoint, QRect, QRectF, QSize, QSizeF
from PyQt5.QtGui import QTransform, QCursor, QPainterPath
from PyQt5.QtWidgets import QGraphicsView, QRubberBand, QMenu, QInputDialog, QMessageBox
from backend.caffe.dict_helper import DictHelper
from gui.main_window.node_editor.items.connection_item import ConnectionItem
from gui.main_window.node_editor.items.connector_item import ConnectorItem
from gui.main_window.node_editor.items.node_item import NodeItem
from gui.main_window.node_editor.node_editor_constants import Constants
class NodeEditorEventHandler:
def __init__(self, nodeEditor, view, scene):
self.nodeEditor = nodeEditor
self.view = view
self.scene = scene
# Declare local variables used in mouse and keyboard events
self.previousGlobalMousePos = QPointF(0.0, 0.0)
self.previousLocalMousePos = QPointF(0.0, 0.0)
self.clickOrigin = QPoint(0, 0)
self.clearSelection = False
self.setSelectionToItem = None
self.removeFromSelection = False
self.removeFromSelectionItemID = None
self.pressedItem = None
# Save the last item the mouse has moved over
self.lastHoveredItem = None
# Save whether a button of the mouse or keyboard is currently being held down by user
self.leftMouseButtonHold = False
self.middleMouseButtonHold = False
self.copySelection = []
self.blockTranslate = False
# Save whether there is an ongoing rubber band selection
self.dragMode = "Drag"
self.dragModeBlocked = False
self.view.setCursor(QCursor(Qt.OpenHandCursor))
self.view.setTransformationAnchor(QGraphicsView.NoAnchor)
# Set standard drag mode and create rubber band element
self.rubberBand = QRubberBand(QRubberBand.Rectangle, view)
# Save if editing is allowed
self.disabled = False
def wheelEvent(self, QGraphicsSceneWheelEvent):
# Handle the wheel event, which changes the zoom of the view
oldScalePos = self.view.mapToScene(QGraphicsSceneWheelEvent.pos())
if QGraphicsSceneWheelEvent.angleDelta().y() > 0:
self.view.scaleSymmetric(Constants.viewScaleFactor)
else:
self.view.scaleSymmetric(1 / Constants.viewScaleFactor)
# Store the position of the mouse with the new scaling factor
newScalePos = self.view.mapToScene(QGraphicsSceneWheelEvent.pos())
# Translate the view so that the mouse is the anchor for the zoom
translationFactor = newScalePos - oldScalePos
self.view.translate(translationFactor.x(), translationFactor.y())
def mousePressEvent(self, QMouseEvent):
# Block all changes of the dragging mode
self.dragModeBlocked = True
# Transform position into scene position
scenePos = self.view.mapToScene(QMouseEvent.pos())
# Take action according to pressed mouse button
if QMouseEvent.button() == Qt.LeftButton:
# Store that left mouse button is held down
self.leftMouseButtonHold = True
# Further actions according to current mouse mode
if self.dragMode == "Drag":
# First of, clear former selection
self.networkManager.clearSelection()
for item in self.scene.selectedItems():
item.setSelected(False)
# Store the global position to calculate translation of view on mouse move
self.previousGlobalMousePos = QMouseEvent.pos()
elif self.dragMode == "Selection":
# First of, clear former selection
self.networkManager.clearSelection()
# Start the rubber band rectangle selection
self.clickOrigin = scenePos
self.rubberBand.setGeometry(QRect(self.view.mapFromScene(self.clickOrigin), QSize()))
self.rubberBand.show()
elif self.dragMode == "Items":
# Selection change when left mouse button is pressed, also prepare for moving selected elements
self.previousLocalMousePos = scenePos
self.pressedItem = self.scene.itemAt(scenePos, QTransform())
if isinstance(self.pressedItem, NodeItem):
#check if a connection is in the selection, if yes, clear it from the selection
for item in self.scene.selectedItems():
if isinstance(item, ConnectionItem):
item.setSelected(False)
# Check if ctrl key is held down
if QMouseEvent.modifiers() & Qt.ControlModifier:
# Check if there are connection elements in the selection, if yes, clear selection
for item in self.scene.selectedItems():
if isinstance(item, ConnectionItem):
self.networkManager.clearSelection()
break
# Check if the item has already been selected
if self.pressedItem.isSelected():
# Start removing from selection. only finish if items are not moved before mouse release
self.removeFromSelection = True
self.removeFromSelectionItemID = self.pressedItem.getLayerID()
else:
self.networkManager.addLayerToSelection(self.pressedItem.getLayerID())
else:
# Check if there is no currently selected Item
if len(self.scene.selectedItems()) == 0:
# Select the new Item
self.networkManager.setSelection(self.pressedItem.getLayerID())
elif len(self.scene.selectedItems()) == 1:
# Check if the item has already been selected
if self.pressedItem.isSelected():
# Start clearing selection, only finish if item is not moved before mouse release
self.clearSelection = True
else:
self.networkManager.setSelection(self.pressedItem.getLayerID())
elif len(self.scene.selectedItems()) > 1:
# Prepare for the item that has been pressed to be the only selected Item, if it is not moved in the meantime
self.setSelectionToItem = scenePos
elif isinstance(self.pressedItem, ConnectorItem):
# Connector has been clicked, start the connection process (further handling in mouse move event)
self.scene.startConnection(self.pressedItem, scenePos)
elif isinstance(self.pressedItem, ConnectionItem):
if QMouseEvent.modifiers() & Qt.ControlModifier:
# Check if there are node items in the selection, if yes, clear selection
if len(self.scene.selectedItems()) > 0:
for item in self.scene.selectedItems():
if isinstance(item, NodeItem):
self.networkManager.clearSelection()
break
if self.pressedItem.isSelected():
self.pressedItem.setSelected(False)
else:
self.pressedItem.setSelected(True)
else:
if self.pressedItem.isSelected():
self.pressedItem.setSelected(False)
else:
self.pressedItem.setSelected(True)
else:
# Check if the item is already selected
if self.pressedItem.isSelected():
if len(self.scene.selectedItems()) > 0:
self.networkManager.clearSelection()
for item in self.scene.selectedItems():
item.setSelected(False)
self.pressedItem.setSelected(True)
else:
self.pressedItem.setSelected(False)
else:
self.networkManager.clearSelection()
for item in self.scene.selectedItems():
item.setSelected(False)
self.pressedItem.setSelected(True)
elif QMouseEvent.button() == Qt.RightButton:
# Store the item that has been pressed by the mouse
pressedItem = self.scene.itemAt(scenePos, QTransform())
if isinstance(pressedItem, NodeItem):
# Make sure item selection is correct
if not pressedItem.isSelected():
if QMouseEvent.modifiers() & Qt.ControlModifier:
self.networkManager.addLayerToSelection(pressedItem.getLayerID())
else:
self.networkManager.setSelection(pressedItem.getLayerID())
contextMenu = QMenu()
renamingAction = None
addTopAction = None
addBottomAction = None
removeAction = None
if not self.disabled:
renamingAction = contextMenu.addAction("Rename")
addTopAction = contextMenu.addAction("Add top")
addBottomAction = contextMenu.addAction("Add bottom")
copyItemsAction = contextMenu.addAction("Copy")
if not self.disabled:
removeAction = contextMenu.addAction("Remove")
action = contextMenu.exec_(QMouseEvent.screenPos().toPoint())
if action is not None:
if action == addTopAction:
name, ok = QInputDialog.getText(self.view, "Add top blob", "Top blob name:")
name = name.strip()
duplicateFound = True # search for duplicates
# check if name is not whitespace and if name is duplicate, if name is wrong keep asking
# until user cancels or enters a valid name
while ok:
if len(name) == 0:
QMessageBox.warning(self.view, "Top blob name is empty",
"The name for the top blob can't be empty or all whitespace characters.")
name, ok = QInputDialog.getText(self.view, "Add top blob", "Top blob name:")
name = name.strip()
elif duplicateFound:
node = self.nodeEditor.getNode(pressedItem.getLayerID())
if len(node.getTopConnectors()) > 0: # if no top connectors, this part ca be skipped
for blob in node.getTopConnectors():
if name == blob.getBlobName():
QMessageBox.warning(self.view, "Top blob name already exists",
"The name you chose already exists on this Layer, please choose another name.")
name, ok = QInputDialog.getText(self.view, "Change name of top blob",
"Top blob name:")
name = name.strip()
duplicateFound = True
break # stop for loop, continue while loop
else:
duplicateFound = False
else:
break # if no other top blob was found
else:
break # if string is valid
if ok:
self.nodeEditor.tryToAddTopBlob(pressedItem.getLayerID(), name)
elif action == addBottomAction:
self.nodeEditor.tryToAddBottomBlob(pressedItem.getLayerID(), "")
elif action == copyItemsAction:
self.nodeEditor.setCopySelection()
elif action == removeAction:
self.networkManager.deleteSelectedLayers()
elif action == renamingAction:
layerParams = DictHelper(self.nodeEditor.getNetworkManager().network).layerParams(pressedItem.getLayerID())
name, ok = QInputDialog.getText(self.view, "Rename this layer", "New name:", text=layerParams["name"])
name = name.strip()
while ok:
if len(name) == 0:
QMessageBox.warning(self.view, "Name is empty",
"The name of the layer can't be empty or all whitespace characters.")
name, ok = QInputDialog.getText(self.view, "Rename this layer", "New name:")
name = name.strip()
else:
layerParams["name"] = name
break
elif isinstance(pressedItem, ConnectionItem):
self.networkManager.clearSelection()
for item in self.scene.selectedItems():
item.setSelected(False)
pressedItem.setSelected(True)
elif pressedItem == None:
# Open a context menu.
contextMenu = QMenu()
# If an item has been copied, show the 'Paste' action.
if len(self.copySelection) > 0 and not self.disabled:
pastAction = contextMenu.addAction("Paste")
contextMenu.addSeparator()
# Add the 'Arrange Network' actions.
horArrangeAction = contextMenu.addAction("Arrange Network Horizontally")
vertArrangeAction = contextMenu.addAction("Arrange Network Vertically")
# Execute the menu and act according to the selected action.
action = contextMenu.exec_(QMouseEvent.screenPos().toPoint())
if action and action.text() == 'Paste':
self.networkManager.duplicateLayers(self.copySelection, self.view.mapToScene(QMouseEvent.pos()))
elif action and action.text() == 'Arrange Network Horizontally':
self.view.arrangeHorizontallyClicked.emit()
elif action and action.text() == 'Arrange Network Vertically':
self.view.arrangeVerticallyClicked.emit()
def mouseMoveEvent(self, QMouseEvent):
# Transform position into scene position
scenePos = self.view.mapToScene(QMouseEvent.pos())
if self.leftMouseButtonHold:
if self.dragMode == "Drag":
# Set the new mouse cursor
self.view.setCursor(QCursor(Qt.ClosedHandCursor))
# Calculate the movement for the view translation
offset = self.previousGlobalMousePos - QMouseEvent.pos()
self.previousGlobalMousePos = QMouseEvent.pos()
offset *= 1/self.view.getScale()
self.view.translate(-offset.x(), -offset.y())
elif self.dragMode == "Selection":
# Check if the user has left the scene with the mouse, if yes, translate accordingly
if QMouseEvent.pos().x() > self.view.viewport().width() and QMouseEvent.pos().y() > self.view.viewport().height():
QCursor.setPos(self.view.mapToGlobal(QPoint(self.view.viewport().width(), self.view.viewport().height())))
self.view.translate(-3 * self.view.getScale(), -3 * self.view.getScale())
elif QMouseEvent.pos().x() < 0 and QMouseEvent.pos().y() < 0:
QCursor.setPos(self.view.mapToGlobal(QPoint(1, 1)))
self.view.translate(3 * self.view.getScale(), 3 * self.view.getScale())
elif QMouseEvent.pos().x() > self.view.viewport().width() and QMouseEvent.pos().y() < 0:
QCursor.setPos(self.view.mapToGlobal(QPoint(self.view.viewport().width(), 1)))
self.view.translate(-3 * self.view.getScale(), 3 * self.view.getScale())
elif QMouseEvent.pos().x() < 0 and QMouseEvent.pos().y() > self.view.viewport().height():
QCursor.setPos(self.view.mapToGlobal(QPoint(1, self.view.viewport().height())))
self.view.translate(3 * self.view.getScale(), -3 * self.view.getScale())
elif QMouseEvent.pos().x() > self.view.viewport().width():
QCursor.setPos(self.view.mapToGlobal(QPoint(self.view.viewport().width(), QMouseEvent.pos().y())))
self.view.translate(-3 * self.view.getScale(), 0)
elif QMouseEvent.pos().y() > self.view.viewport().height():
QCursor.setPos(self.view.mapToGlobal(QPoint(QMouseEvent.pos().x(), self.view.viewport().height())))
self.view.translate(0, -3 * self.view.getScale())
elif QMouseEvent.pos().x() < -1:
QCursor.setPos(self.view.mapToGlobal(QPoint(1, QMouseEvent.pos().y())))
self.view.translate(3 * self.view.getScale(), 0)
elif QMouseEvent.pos().y() < -1:
QCursor.setPos(self.view.mapToGlobal(QPoint(QMouseEvent.pos().x(), 1)))
self.view.translate(0, 3 * self.view.getScale())
# Map origin coordinates to current view coordinates
clickOriginView = self.view.mapFromScene(self.clickOrigin)
# Calculate the new top left point as well as width and height
topLeft = QPoint(min(clickOriginView.x(), self.view.mapFromGlobal(QCursor.pos()).x()),
min(clickOriginView.y(), self.view.mapFromGlobal(QCursor.pos()).y()))
size = QSize(abs(clickOriginView.x() - self.view.mapFromGlobal(QCursor.pos()).x()),
abs(clickOriginView.y() - self.view.mapFromGlobal(QCursor.pos()).y()))
self.rubberBand.setGeometry(QRect(topLeft, size))
elif self.dragMode == "Items":
if self.scene.currentlyConnecting is not None and not self.disabled:
self.scene.currentlyConnecting.updateMousePosition(scenePos)
else:
# Moving should only be enabled for node items
if isinstance(self.pressedItem, NodeItem):
if self.pressedItem.isSelected():
# Cancel the selection clearing
if self.clearSelection:
self.clearSelection = False
elif self.removeFromSelection:
self.removeFromSelection = False
self.removeFromSelectionItemID = None
elif self.setSelectionToItem is not None:
self.setSelectionToItem = None
else: # if the selected item was not selected before
self.networkManager.clearSelection()
self.networkManager.setSelection(
self.scene.itemAt(self.setSelectionToItem, QTransform()).getLayerID())
self.clearSelection = False
self.removeFromSelection = False
self.removeFromSelectionItemID = None
self.setSelectionToItem = None
# Calculate the change in mouse position
currentLocalMousePos = scenePos
translation = currentLocalMousePos - self.previousLocalMousePos
self.previousLocalMousePos = currentLocalMousePos
# Move all selected Elements according to mouse movement
resetMouse = self.translateItems(translation.x(), translation.y())
if resetMouse:
QCursor.setPos(QCursor.pos().x() - translation.x(), QCursor.pos().y() - translation.y())
else:
if self.lastHoveredItem != self.scene.itemAt(scenePos, QTransform()):
# If no button is held down, change mouse pointer according to item it hovers over
if self.scene.itemAt(scenePos, QTransform()) is None:
if QMouseEvent.modifiers() & Qt.ControlModifier:
self.view.setCursor(QCursor(Qt.CrossCursor))
self.dragMode = "Selection"
else:
self.view.setCursor(QCursor(Qt.OpenHandCursor))
self.dragMode = "Drag"
else:
self.view.setCursor(QCursor(Qt.ArrowCursor))
self.dragMode = "Items"
# Set a new last hovered item
self.lastHoveredItem = self.scene.itemAt(scenePos, QTransform())
def mouseReleaseEvent(self, QMouseEvent):
# Transform position into scene position
scenePos = self.view.mapToScene(QMouseEvent.pos())
if QMouseEvent.button() == Qt.LeftButton:
# Store that button is no longer being held down
self.leftMouseButtonHold = False
if self.dragMode == "Drag":
self.view.setCursor(Qt.OpenHandCursor)
elif self.dragMode == "Selection":
# Disable the rubber band selection and hide it
self.isRubberBandSelection = False
self.rubberBand.hide()
# Set the scene selection to all elements within the rubber band area
painterPath = QPainterPath()
# Calculate rectangle selection position, using the rubber band will fail at this point
clickOriginView = self.view.mapFromScene(self.clickOrigin)
sceneOrigin = self.view.mapToScene(clickOriginView)
topLeft = QPointF(min(sceneOrigin.x(), scenePos.x()),
min(sceneOrigin.y(), scenePos.y()))
size = QSizeF(abs(sceneOrigin.x() - scenePos.x()),
abs(sceneOrigin.y() - scenePos.y()))
# Select every element within the rectangle selection
painterPath.addRect(QRectF(topLeft, size))
self.scene.setSelectionArea(painterPath, QTransform())
# Get the ids of all selected elements to set selection to other ui elements
selectedIDs = []
# Kick all connection items out of that selection
for item in self.scene.selectedItems():
if isinstance(item, NodeItem):
selectedIDs.append(item.getLayerID())
item.setSelected(False)
self.networkManager.setSelectionList(selectedIDs)
elif self.dragMode == "Items":
if self.scene.currentlyConnecting is not None:
self.scene.connectCurrentConnection(scenePos)
elif isinstance(self.pressedItem, ConnectorItem):
for item in self.scene.selectedItems():
item.setSelected(False)
self.networkManager.clearSelection()
for connection in self.pressedItem.getConnections():
connection.setSelected(True)
elif self.clearSelection:
self.networkManager.clearSelection()
self.clearSelection = False
elif self.removeFromSelection and self.removeFromSelectionItemID is not None:
self.networkManager.removeLayerFromSelection(self.removeFromSelectionItemID)
self.removeFromSelection = False
self.removeFromSelectionItemID = None
elif self.setSelectionToItem is not None:
self.networkManager.setSelection(self.scene.itemAt(self.setSelectionToItem, QTransform()).getLayerID())
self.setSelectionToItem = None
# Unblock the drag mode
self.dragModeBlocked = False
# Change the cursor type according to the bool value
if self.lastHoveredItem is None:
if QMouseEvent.modifiers() & Qt.ControlModifier:
self.view.setCursor(QCursor(Qt.CrossCursor))
self.dragMode = "Selection"
else:
self.view.setCursor(QCursor(Qt.OpenHandCursor))
self.dragMode = "Drag"
else:
self.view.setCursor(QCursor(Qt.ArrowCursor))
self.dragMode = "Items"
def keyPressEvent(self, QKeyEvent):
if QKeyEvent.key() == Qt.Key_Control:
self.setCrosshairState(True)
else:
if QKeyEvent.key() == Qt.Key_Up:
self.translateItems(0, - 5 * self.view.getScale())
if QKeyEvent.key() == Qt.Key_Down:
self.translateItems(0, 5 * self.view.getScale())
if QKeyEvent.key() == Qt.Key_Right:
self.translateItems(5 * self.view.getScale(), 0)
if QKeyEvent.key() == Qt.Key_Left:
self.translateItems(- 5 * self.view.getScale(), 0)
if QKeyEvent.key() == Qt.Key_C:
if QKeyEvent.modifiers() & Qt.ControlModifier:
self.copySelection = []
for item in self.scene.selectedItems():
# If getLayerId gives an error, do not copy the element
try:
self.copySelection.append(item.getLayerID())
except AttributeError:
None
if QKeyEvent.key() == Qt.Key_V and not self.disabled:
if len(self.copySelection) > 0:
self.networkManager.duplicateLayers(self.copySelection)
def keyReleaseEvent(self, QKeyEvent):
if QKeyEvent.key() == Qt.Key_Control:
self.setCrosshairState(False)
elif QKeyEvent.key() == Qt.Key_Delete and not self.disabled:
self.networkManager.deleteSelectedLayers()
def setNetworkManager(self, networkManager):
self.networkManager = networkManager
def setCrosshairState(self, ctrlPressed):
# Only change state when user is not currently selecting anything
if not self.dragModeBlocked:
# Change the cursor type according to the bool value
if self.lastHoveredItem is None:
if ctrlPressed:
self.view.setCursor(QCursor(Qt.CrossCursor))
self.dragMode = "Selection"
else:
self.view.setCursor(QCursor(Qt.OpenHandCursor))
self.dragMode = "Drag"
else:
self.view.setCursor(QCursor(Qt.ArrowCursor))
self.dragMode = "Items"
def translateItems(self, deltaX, deltaY):
# Moving should only be enabled for node items
if self.scene.selectedItems() and isinstance(self.scene.selectedItems()[0], NodeItem):
# Prevent multiple layers from changing relative positions when the scenes border is reached
selectedItems = self.scene.selectedItems()
if len(selectedItems) > 0:
selectedItemsBoundingRect = selectedItems[0].sceneBoundingRect()
for i in range(1, len(selectedItems)):
selectedItemsBoundingRect = selectedItemsBoundingRect.united(selectedItems[i].sceneBoundingRect())
# Fix for wrong scene bounding rects right and bottom
selectedItemsBoundingRect.setWidth(selectedItemsBoundingRect.width() + 3)
selectedItemsBoundingRect.setHeight(selectedItemsBoundingRect.height() + 3)
newBoundingRect = QRectF(selectedItemsBoundingRect)
newBoundingRect.moveTo(selectedItemsBoundingRect.topLeft() + QPointF(deltaX, deltaY))
sceneRect = self.scene.sceneRect()
moved = False
if newBoundingRect.left() < sceneRect.left():
deltaX = sceneRect.left() - selectedItemsBoundingRect.left()
moved = True
elif newBoundingRect.right() > sceneRect.right():
deltaX = sceneRect.right() - selectedItemsBoundingRect.right()
moved = True
if newBoundingRect.top() < sceneRect.top():
deltaY = sceneRect.top() - selectedItemsBoundingRect.top()
moved = True
elif newBoundingRect.bottom() > sceneRect.bottom():
deltaY = sceneRect.bottom() - selectedItemsBoundingRect.bottom()
moved = True
for item in self.scene.selectedItems():
item.setPos(item.scenePos() + QPointF(deltaX, deltaY))
# Check if any of the objects have reached the boundary and the view should be translated
if len(selectedItems) > 0:
selectedItemsBoundingRect = selectedItems[0].sceneBoundingRect()
for i in range(1, len(selectedItems)):
selectedItemsBoundingRect = selectedItemsBoundingRect.united(
selectedItems[i].sceneBoundingRect())
newRect = self.view.mapFromScene(selectedItemsBoundingRect).boundingRect()
if newRect.left() < 0 or newRect.right() > self.view.viewport().width() or newRect.top() < 0 or newRect.bottom() > self.view.viewport().height():
self.view.translate(-deltaX, -deltaY)
moved = True
return moved
def setCopySelection(self):
self.copySelection = []
for item in self.scene.selectedItems():
self.copySelection.append(item.getLayerID())
def setDisabled(self, disabled):
self.disabled = disabled
Barista/gui/main_window/node_editor/node_editor_scene.py 0000664 ڊ0011117 00000025324 13265130053 024006 0 ustar klemms o0cvpr from PyQt5.QtCore import QLineF
from PyQt5.QtGui import QPen, QColor, QCursor, QTransform
from PyQt5.QtWidgets import QGraphicsScene, QMessageBox
from gui.main_window.node_editor.items.connection_item import ConnectionItem
from gui.main_window.node_editor.items.connector_item import ConnectorItem
from gui.main_window.node_editor.items.node_item import NodeItem
from gui.main_window.node_editor.node_editor_constants import Constants
class NodeEditorScene(QGraphicsScene):
def __init__(self, view, initialWidth, initialHeight, nodeEditor):
super(NodeEditorScene, self).__init__(-(initialWidth / 2), -(initialHeight / 2), initialWidth, initialHeight, view)
self.__view = view
self.__nodeEditor = nodeEditor
self.currentlyConnecting = None
self.disabled = False
# Create the event handler for this scene
self.eventHandler = None
def setEventHandler(self, eventHandler):
self.eventHandler = eventHandler
def getNodeEditor(self):
return self.__nodeEditor
def getView(self):
return self.__view
def startConnection(self, connector, scenePosition):
""" Creates a new connection item, connected to the connector """
self.currentlyConnecting = ConnectionItem(self.__nodeEditor)
self.currentlyConnecting.setConnector(connector)
self.currentlyConnecting.updateMousePosition(scenePosition)
self.addItem(self.currentlyConnecting)
def connectCurrentConnection(self, position):
""" Try to connect the currently connecting connection.
If no valid connector was found, the connection disappears """
# get the item at the position
itemAt = self.itemAt(position.toPoint(), self.getView().transform())
# remove the connection (a new connection will get added if there is a valid connector)
self.removeItem(self.currentlyConnecting)
connection = self.currentlyConnecting
self.currentlyConnecting = None
""" if itemAt is a Connector (Top/Bottom) item (if you pull onto a Blob) """
if itemAt is not None and isinstance(itemAt, ConnectorItem) and not self.disabled:
# check, whether the connection is already connected to connector of the given type (top/bottom)
if connection.checkSameConnectorTypeRestriction(itemAt):
# get the connectors
if itemAt.isTopConnector():
topConnector = itemAt
bottomConnector = connection.getConnectorIfNotFullyConnected()
else:
topConnector = connection.getConnectorIfNotFullyConnected()
bottomConnector = itemAt
# get data needed to notify the underling data structure
topLayerID = topConnector.getNodeItem().getLayerID()
bottomLayerID = bottomConnector.getNodeItem().getLayerID()
topBlobIndex = topConnector.getIndex()
bottomBlobIndex = bottomConnector.getIndex()
# notify to change the data
self.__nodeEditor.tryToConnect(topLayerID, topBlobIndex, bottomLayerID, bottomBlobIndex)
""" if itemAt is a Node Item (if you pull onto a layer) """
if itemAt is not None and isinstance(itemAt, NodeItem) and not self.disabled:
# test if connector starts at a top Blob
if connection.getConnectorIfNotFullyConnected().isTopConnector():
# bottomNode is itemAt
bottomNode = itemAt
# get layer IDs
topLayerID = connection.getConnectorIfNotFullyConnected().getNodeItem().getLayerID()
bottomLayerID = bottomNode.getLayerID()
topBlobIndex = connection.getConnectorIfNotFullyConnected().getIndex()
# get the Index of the new Blob, should it be necessary to create one
# (determined in the following for loop)
bottomBlobIndex = bottomNode.getBottomConnectorCount()
# current connection top name and phase
topBlobName = connection.getConnectorIfNotFullyConnected().getBlobName()
topBlobPhase = connection.getConnectorIfNotFullyConnected().getPhase()
# check if there is a connected Node that has a different phase than the currently
# connecting Node, but has a connection with the same top Blob Name
topBlobFound = False
for bottomBlob in bottomNode.getBottomConnectors():
if len(bottomBlob.getConnectedNodes()) > 0:
for topNode in bottomBlob.getConnectedNodes():
for topBlob in topNode.getTopConnectors():
if topBlob.getBlobName() == topBlobName and topBlob.getPhase() != topBlobPhase:
bottomBlobIndex = bottomBlob.getIndex()
topBlobFound = True
break
# otherwise (if no corresponding top Blob was found)
# get Index of first empty bottom blob (if available)
counter = -1
emptyBlobAvailable = False
if not topBlobFound:
for blob in bottomNode.getBottomConnectors():
counter += 1
if len(blob.getConnectedNodes()) == 0:
bottomBlobIndex = counter
emptyBlobAvailable = True
break
# add empty bottom blob property
if not emptyBlobAvailable and not topBlobFound:
self.__nodeEditor.tryToAddBottomBlob(bottomLayerID, "")
# connect nodes
connected = self.__nodeEditor.tryToConnect(topLayerID, topBlobIndex, bottomLayerID, bottomBlobIndex)
# if the connection did not work but a new blob was created, remove it
if not connected and not emptyBlobAvailable and not topBlobFound:
bottomNode.removeBottomConnector(bottomBlobIndex)
def dragEnterEvent(self, event):
""" Verifies that the data dragged into the widget is correct """
mimeData = event.mimeData().text()
if not self.__nodeEditor.getNetworkManager().isValidLayerType(mimeData):
event.ignore()
def dragMoveEvent(self, event):
""" Empty reimplementation """
def dropEvent(self, event):
mimeData = event.mimeData().text()
if self.__nodeEditor.getNetworkManager().isValidLayerType(mimeData):
# When calculating the replace option, use QCursor because pos() of event is (0, 0) for some reason
item = self.itemAt(self.__view.mapToScene(self.__view.mapFromGlobal(QCursor.pos())), QTransform())
if isinstance(item, NodeItem):
if QMessageBox.warning(self.__view, "Replace Layer",
"Would you like to replace " + item.getName() + " with the new layer?",
QMessageBox.Yes, QMessageBox.No) == QMessageBox.No:
item = None
# Create the new node item
self.__nodeEditor.tryToCreateLayer(mimeData, event.scenePos().x(), event.scenePos().y())
# Check if the user wants to replace an old layer with this one
if item != None:
# Get the new layer id
newItem = self.selectedItems()[0]
# Add top connectors
for i in range(0, item.getTopConnectorCount()):
self.__nodeEditor.tryToAddTopBlob(newItem.getLayerID(), item.getTopConnectors()[i].getBlobName())
for j in range(item.getTopConnectors()[i].getConnectionCount()-1, -1, -1):
connection = item.getTopConnectors()[i].getConnections()[j]
connection.setTopConnector(newItem.getTopConnectors()[i])
item.getTopConnectors()[i].removeConnection(connection)
newItem.getTopConnectors()[i].addConnection(connection)
# Add bottom connectors
for i in range(0, item.getBottomConnectorCount()):
self.__nodeEditor.tryToAddBottomBlob(newItem.getLayerID(), item.getBottomConnectors()[i].getBlobName())
for j in range(item.getBottomConnectors()[i].getConnectionCount()-1, -1, -1):
connection = item.getBottomConnectors()[i].getConnections()[j]
connection.setBottomConnector(newItem.getBottomConnectors()[i])
item.getBottomConnectors()[i].removeConnection(connection)
newItem.getBottomConnectors()[i].addConnection(connection)
# Translate new item to the exact place of the old item
newItem.setPos(item.x(), item.y())
self.__nodeEditor.getNetworkManager().deleteLayer(item.getLayerID())
# Set keyboard focus to the node prototxt_editor view
self.__nodeEditor.setKeyboardFocus()
event.accept()
def drawBackground(self, painter, rect):
""" Draws a large and a small grid on the scene """
super(NodeEditorScene, self).drawBackground(painter, rect)
# SMALL grid
# do not draw small grid if zoomed out
smallGridSize = Constants.sceneGridSize
if self.getView().getScale() < smallGridSize / 10:
self.drawGrid(rect, smallGridSize, QColor(230, 230, 230), painter)
# LARGE grid
# do not draw large grid if zoomed out
largeGridSize = Constants.sceneGridSize * Constants.sceneLargeGridCells
if self.getView().getScale() < largeGridSize / 10:
self.drawGrid(rect, largeGridSize, QColor(200, 200, 200), painter)
def drawGrid(self, rect, gridSize, color, painter):
""" Draw visible grid lines in the current view rect
Concept of grid drawing by Bitto:
http://www.qtcentre.org/threads/5609-Drawing-grids-efficiently-in-QGraphicsScene"""
lines = list()
startX = rect.left() - (rect.left() % gridSize)
startY = rect.top() - (rect.top() % gridSize)
# add vertical lines
x = startX
while x < rect.right():
lines.append(QLineF(x, rect.top(), x, rect.bottom()))
x += gridSize
# add horizontal lines
y = startY
while y < rect.bottom():
lines.append(QLineF(rect.left(), y, rect.right(), y))
y += gridSize
# draw the lines
painter.setPen(QPen(color))
painter.drawLines(lines)
def disableEditing(self, disable):
""" Disable automatic generation of new blobs. """
self.disabled = disable
Barista/gui/main_window/node_editor/node_editor_view.py 0000664 ڊ0011117 00000004515 13265130053 023662 0 ustar klemms o0cvpr from PyQt5.QtWidgets import QGraphicsView
from PyQt5.QtCore import Qt, QPoint, pyqtSignal
from gui.main_window.node_editor.node_editor_constants import Constants
from gui.main_window.node_editor.node_editor_scene import NodeEditorScene
class NodeEditorView(QGraphicsView):
# These signals are emitted from within the NodeEditorEventHandler when the
# actions are clicked from the context menu.
arrangeVerticallyClicked = pyqtSignal()
arrangeHorizontallyClicked = pyqtSignal()
def __init__(self, nodeEditor, parent=None):
super(NodeEditorView, self).__init__(parent)
Constants.setPaintDevice(self)
self.__scene = NodeEditorScene(self, Constants.sceneInitialWidth, Constants.sceneInitialHeight, nodeEditor)
self.setScene(self.__scene)
self.setTransformationAnchor(QGraphicsView.AnchorUnderMouse)
self.setViewportUpdateMode(QGraphicsView.FullViewportUpdate)
self.eventHandler = None
self.centerOn(0, 0)
def setEventHandler(self, eventHandler):
self.eventHandler = eventHandler
def scaleSymmetric(self, scaleFactor):
"""
set scale and zooms the view, regarding the max and min zoom
"""
scale = self.getScale()*scaleFactor
# Prevent over scale
if scale > Constants.viewMinScale:
factor = (1/self.getScale())*Constants.viewMinScale
self.scale(factor, factor)
return
elif scale < Constants.viewMaxScale:
factor = (1/self.getScale())*Constants.viewMaxScale
self.scale(factor, factor)
return
# scale
self.scale(scaleFactor, scaleFactor)
def getScale(self):
return self.transform().m11()
def wheelEvent(self, event):
self.eventHandler.wheelEvent(event)
def mouseDoubleClickEvent(self, QMouseEvent):
return
def mousePressEvent(self, QMouseEvent):
self.eventHandler.mousePressEvent(QMouseEvent)
def mouseMoveEvent(self, QMouseEvent):
self.eventHandler.mouseMoveEvent(QMouseEvent)
def mouseReleaseEvent(self, QMouseEvent):
self.eventHandler.mouseReleaseEvent(QMouseEvent)
def keyPressEvent(self, QKeyEvent):
self.eventHandler.keyPressEvent(QKeyEvent)
def keyReleaseEvent(self, QKeyEvent):
self.eventHandler.keyReleaseEvent(QKeyEvent)
Barista/gui/main_window/node_editor/node_sort.py 0000664 ڊ0011117 00000021022 13265130053 022321 0 ustar klemms o0cvpr from backend.barista.utils.logger import Log
from gui.main_window.node_editor.items.node_item import NodeItem
# TODO:
# -try avoiding intersecting connections (arrange ordered by connectors)
callerId = Log.getCallerId('GraphLayouter')
class NodeSort():
class Node:
OFFSET = 50
CENTER_X = 0
CENTER_Y = 0
def __init__(self, guiNode, listOfConnectedNodes):
self.guiNode = guiNode
# contains only output nodes of type Node
self.listOfConnectedNodes = listOfConnectedNodes
self.height = self.guiNode.boundingRect().height()
self.width = self.guiNode.boundingRect().width()
self.x = -1 # mark pos not set yet
self.y = -1 # mark pos not set yet
self.visited = False
def setPosition(self, columnWidths, rowHeights):
xpos = -NodeSort.Node.CENTER_X
for x in range(0, self.x):
xpos += columnWidths[x]+self.OFFSET
ypos = -NodeSort.Node.CENTER_Y
for y in range(0, self.y):
ypos += rowHeights[y]+self.OFFSET
self.guiNode.setPos(xpos, ypos)
# Special methods, not really needed noticed to late,
# handles only if there is a connection between topand bottom
def getIndexWhereTopEqualsBottom(self):
'''
Needed for special cases where the top is
equal to the bottom (where input is equal to output)
'''
if (len(self.listOfConnectedNodes) > 0 and
self in self.listOfConnectedNodes):
return self.listOfConnectedNodes.index(self)
return -1
def getOutputNodesWithoutSelf(self):
nodes = self.listOfConnectedNodes
index = self.getIndexWhereTopEqualsBottom()
if(index != -1):
# remove index from list
return nodes[:index] + nodes[index+1:]
return nodes
def __init__(self, layers, view, vertical=False):
self.vertical = vertical
if(vertical):
NodeSort.Node.OFFSET = 70
else:
NodeSort.Node.OFFSET = 30
self.__hasCycle = False
self.columnWidths = []
self.rowHeights = []
# all abstract nodes (type Node) that exist
self.nodes = []
# all gui nodes that exist
self.guiNodes = []
if(type(layers) is list):
Log.error("wrong type of layers", callerId)
return
# fills nodes and guiNodes
self.getAbstractNodes(layers.values())
if(self.__hasCycle):
Log.error("There is a Cycle in that graph, cannot be handled yet", callerId)
return
if(not layers.values()):
return
self.sort(self.nodes, view)
def getAbstractNodes(self, graphItems):
for graphItem in graphItems:
if(type(graphItem) is NodeItem):
self.createNodeItem(graphItem, [])
def createNodeItem(self, guiNode, visitedGuiNodes):
# if node not handled yet, add it
if(guiNode not in self.guiNodes):
outputNodes = []
# every connector can hold more connections
connectedGuiNodes = []
for guiConnectorItem in guiNode.getTopConnectors():
connectedGuiNodes += guiConnectorItem.getConnectedNodes()
for guiNodeItem in connectedGuiNodes:
if(guiNodeItem in visitedGuiNodes):
# TODO: GuiNode.setErrorMArker()
self.__hasCycle = True
return
newVisitedNodes = list(visitedGuiNodes)
newVisitedNodes.append(guiNodeItem)
if guiNodeItem not in self.guiNodes:
self.createNodeItem(guiNodeItem, newVisitedNodes)
if(self.__hasCycle):
return
outputNodes.append(self.nodes[self.guiNodes.index(guiNodeItem)])
node = NodeSort.Node(guiNode, outputNodes)
self.nodes.append(node)
self.guiNodes.append(guiNode)
def sort(self, listOfNodes, view):
firstNodes = self.getListOfFirstNodes(listOfNodes)
# just reset, firstNOdes used this
self.setAllUnvisited(listOfNodes)
self.stepSize = 1
self.x = 0
self.y = 0
self.maxY = 0
self.maxX = 0
self.traverseConnectedNodes(firstNodes, 0)
self.setCellSizes(listOfNodes)
if(self.vertical):
self.verticalGrid()
self.setViewBounds(view)
self.updateGuiNodes(listOfNodes)
def getListOfFirstNodes(self, listOfNodes):
firstNodes = []
for ele in listOfNodes:
ele.visited = True
for ele2 in listOfNodes:
if(ele in ele2.listOfConnectedNodes):
ele.visited = False
break
if(ele.visited):
firstNodes.append(ele)
return firstNodes
def setAllUnvisited(self, listOfNodes):
for ele in listOfNodes:
ele.visited = False
def traverseConnectedNodes(self, nodes, x):
y = 0
for ele in nodes:
# ele could be visited before, so just if it was not good (to low) calculated reset it
if ele.x < x:
ele.x = x
if ele.y < y:
# if ele.x changed there must be recalculation for y
y = self.getCurrentYMaxInColumn(ele.x)
ele.y = y
# maximum for maybe needed for make it pretty
if self.maxX < ele.x:
self.maxX = ele.x
if self.maxY < ele.y:
self.maxY = ele.y
self.traverseConnectedNodes(ele.listOfConnectedNodes, ele.x+self.stepSize)
def getCurrentYMaxInColumn(self, x):
nodesAtX = self.getNodesInColumn(self.nodes, x)
if(len(nodesAtX) != 0):
maxY = max(node.y for node in nodesAtX) + self.stepSize
else:
maxY = 0
return maxY
def updateGuiNodes(self, listOfNodes):
# set Position for all nodes
for node in listOfNodes:
node.setPosition(self.columnWidths, self.rowHeights)
def getNodesInColumn(self, nodes, x):
nodesInColumnX = []
for ele in nodes:
if(ele.x == x):
nodesInColumnX.append(ele)
return nodesInColumnX
def getNodesInRow(self, nodes, y):
nodesInRowY = []
for ele in nodes:
if(ele.y == y):
nodesInRowY.append(ele)
return nodesInRowY
def setCellSizes(self, nodes):
# Set Column Width
for x in range(0, self.maxX+self.stepSize):
# maxX included because its the maximum USED
nodesAtX = self.getNodesInColumn(nodes, x)
# get maxwidth, every node in the column should have this columnwidth
maxWidth = max(node.width for node in nodesAtX)
if(self.vertical):
maxWidth = max(node.height for node in nodesAtX)
self.columnWidths.append(maxWidth)
# Set Row Height
for y in range(0, self.maxY+self.stepSize):
# maxY included because its the maximum USED
nodesAtY = self.getNodesInRow(nodes, y)
# get Maxheight, every node in the row should have this rowheight
if(len(nodesAtY) != 0):
maxHeight = max(node.height for node in nodesAtY)
if(self.vertical):
maxHeight = max(node.width for node in nodesAtY)
else:
maxHeight = -1
self.rowHeights.append(maxHeight)
def setViewBounds(self, view):
neededWidth = sum(self.columnWidths)+NodeSort.Node.OFFSET*len(self.columnWidths)
neededHeight = sum(self.rowHeights)+NodeSort.Node.OFFSET*len(self.rowHeights)
NodeSort.Node.CENTER_X = int(round(neededWidth/2))
NodeSort.Node.CENTER_Y = int(round(neededHeight/2))
marginScreen = -(NodeSort.Node.OFFSET/2)
x = marginScreen-NodeSort.Node.CENTER_X
y = marginScreen-NodeSort.Node.CENTER_Y
view.fitInView(x, y, neededWidth, neededHeight, True)
# doesnt scale anything, only if there is too much zoom
view.scaleSymmetric(1)
# Layout vertical
def verticalGrid(self):
for node in self.nodes:
# swap x and y position
tmp = node.x
node.x = node.y
node.y = tmp
tmp = self.columnWidths
self.columnWidths = self.rowHeights
self.rowHeights = tmp
Barista/backend/barista/__init__.py 0000664 ڊ0011117 00000000000 13265130053 017477 0 ustar klemms o0cvpr Barista/backend/barista/caffe_versions.py 0000664 ڊ0011117 00000006576 13265130053 020764 0 ustar klemms o0cvpr import os
from backend.barista.utils.logger import LogCaller
versions = []
restart = False
def saveVersions(path=""):
"""Saves the versions to file."""
import pickle
try:
with open(os.path.join(path, "caffeVersions"), "w") as outfile:
global versions
pickle.dump(versions, outfile)
return True
except IOError as e:
return False
def loadVersions(path=""):
"""Reloads the versions from file."""
import pickle
try:
with open(os.path.join(path, "caffeVersions"), "r") as infile:
global versions
versions = pickle.load(infile)
return True
except IOError as e:
return False
def getAvailableVersions():
"""Returns all to barista available caffe versions."""
global versions
return versions
def addVersion(version, path=""):
"""Adds a caffe version to barista.
Returns True if the version was successfully added
Returns False if version could not be added"""
try:
global versions
versions.index(version)
return False
except Exception as e:
versions.append(version)
return saveVersions(path)
def removeVersion(version, path=""):
"""Removes a caffe version from barista.
Returns True if the version was successfully removed
Returns False if version could not be removed"""
try:
global versions
versions.remove(version)
return saveVersions(path)
except Exception as e:
return False
def versionCount():
"""Returns the number of stored caffe versions"""
global versions
return len(versions)
def getVersionByName(name):
"""Returns the version object depending on the given name"""
global versions
for version in versions:
if version.getName() == name:
return version
return None
def getDefaultVersion():
"""Returns the default version object"""
if versionCount() > 0:
global versions
return versions[0]
else:
return None
def setDefaultVersion(name, path=""):
"""Sets the default caffe version of barista,
implemented as beeing the first element of the list
Returns True if the default version was successfully set
Returns False if not"""
version = getVersionByName(name)
if version != None:
global versions
index = versions.index(version)
temp = versions[0]
versions[0] = versions[index]
versions[index] = temp
return saveVersions(path)
else:
return False
class caffeVersion():
"""This class represents a specific caffe version."""
def __init__(self, name, root, binary, python, proto):
self.name = name
self.root = root
self.binary = binary
self.python = python
self.proto = proto
def getName(self):
return self.name
def getRootpath(self):
return self.root
def getBinarypath(self):
return self.binary
def getPythonpath(self):
return self.python
def getProtopath(self):
return self.proto
def setName(self, name):
self.name = name
def setName(self, root):
self.root = root
def setBinarypath(self, binary):
self.binary = binary
def setPythonpath(self, python):
self.python = python
def setProtopath(self, proto):
self.proto = proto Barista/backend/barista/deployed_net.py 0000664 ڊ0011117 00000033716 13265130053 020437 0 ustar klemms o0cvpr import copy
import os
from shutil import copyfile
import backend.caffe.proto_info as info
from backend.barista.utils.logger import Log
from backend.caffe import loader
from backend.caffe import saver
from backend.caffe.dict_helper import DictHelper
from gui.input_manager.database_object import DatabaseObject
class DeployedNet:
"""Based on a given network dictionary, an instance of this class (creates and) provides all data necessary to
export a deployed version of the network.
The applied rules are based on the information provided in the following caffe wiki page:
https://github.com/BVLC/caffe/wiki/Using-a-Trained-Network:-Deploy
In summary, the following actions are performed automatically:
- remove all data layers
- remove all layers that require any labels
- insert new input layers
- append a new Softmax layer (only if none does exist yet and a SoftmaxWithLoss layer used to be included)
Finally, by calling saveProtoTxtFile() the following is done:
- copy the modified prototxt file and the associated caffe model file to the
Restrictions:
- Each data layer must not have more than two top blobs (a general restriction of caffe). Raises a warning.
- At least the label blob name must follow the naming conventions (so it's always called "label"). Raises a warning.
- The shape of a Data layer can only be determined automatically, if the layer type is either "Data"
(LMDB or LEVELDB) or "HDF5Data". Otherwise, a warning will inform the user about necessary manual changes.
"""
def __init__(self, netPrototxtContents):
"""Create a deployment version of the given network.
netPrototxtContents: string
The contents of the prototxt file to deploy.
"""
# create a logger id
self._logId = Log.getCallerId('deployment')
self._originalNetworkDictionary = loader.loadNet(netPrototxtContents)
# init further attributes
self._dataLayers = dict() # a dictionary containing all data layers. keys are the layer ids.
self._labelBlobNames = [] # a list containing all names of blobs, that represent any labels
self._dataBlobNames = [] # a list containing all names of blobs, that represent any input data
self._inputBlobShapes = dict() # the keys of this dictionary equal self._dataBlobNames.
self._deployedNetworkDictionary = copy.deepcopy(self._originalNetworkDictionary)
self._dHelper = DictHelper(self._deployedNetworkDictionary)
# start deployment
self._createDeployedNetwork()
def _createDeployedNetwork(self):
"""Create a deployed version of self._originalNetworkDictionary and save it in self._deployedNetworkDictionary.
Requires self._originalNetworkDictionary to be set before.
"""
# ensure that at least one layer does exist
if len(self._deployedNetworkDictionary["layers"]) <= 0:
raise ValueError("Can not create deployment network, because the original one does not contain any layers.")
self._searchDataLayers()
self._removeDataLayers()
self._removeLayersUsingLabels()
self._insertInputLayers()
self._addSoftmax()
def getProtoTxt(self):
"""return the content of self._deployedNetworkDictionary as a prototxt string."""
return saver.saveNet(self._deployedNetworkDictionary)
def _searchDataLayers(self):
"""Search for all data layers in self._originalNetworkDictionary and store them in self._dataLayers.
Additionally, the names of all top blobs containing label data will be saved in self._labelBlobNames.
"""
nLabelBlobUndetermined = 0 # check whether some label blobs couldn't be determined automatically
nTooManyTopBlobs = 0 # check whether some data layers do have too many top blobs
for id, layer in self._originalNetworkDictionary["layers"].iteritems():
if layer["type"].isDataLayer():
self._dataLayers[id] = layer
# validate number of top blobs
if "top" in layer["parameters"] and len(layer["parameters"]["top"]) > 2:
nTooManyTopBlobs += 1
# remember the associated label blob name
labelBlobName = self._getLabelBlobName(layer)
if labelBlobName is not None and labelBlobName not in self._labelBlobNames:
self._labelBlobNames.append(labelBlobName)
elif labelBlobName is None:
nLabelBlobUndetermined += 1
# remember the associated data blob name and its shape
dataBlobName = self._getDataBlobName(layer)
if dataBlobName is not None and dataBlobName not in self._dataBlobNames:
self._dataBlobNames.append(dataBlobName)
# calculate the shape (this assumes that data layers with the same name are using the same shape)
blobShape = [] # will be left empty, if shape cannot be calculated automatically
if layer["type"].name() in ["Data", "HDF5Data"]:
if layer["type"].name() == "Data":
path = layer["parameters"].get("data_param", {}).get("source")
type = layer["parameters"].get("data_param", {}).get("backend")
elif layer["type"].name() == "HDF5Data":
path = layer["parameters"].get("hdf5_data_param", {}).get("source")
type = "HDF5TXT"
if path is not None and type is not None:
db = DatabaseObject()
db.openFromPath(path, type)
blobShapeTupel = db.getDimensions()
if blobShapeTupel is not None:
blobShapeTupel = blobShapeTupel.get(dataBlobName)
if blobShapeTupel is not None:
blobShape = list(blobShapeTupel)
self._inputBlobShapes[dataBlobName] = blobShape
# show warning, if some label blobs do not have the correct name
if nLabelBlobUndetermined > 0:
Log.log("{} data layers might have been handled incorrectly, because their top blobs are named "
"unconventionally. Please change the name of the blobs which provide labels to \"label\".".format(
nLabelBlobUndetermined
), self._logId)
# show warning, if some data layers have too many top blobs
if nTooManyTopBlobs > 0:
Log.log("{} data layers have too many top blobs. The native caffe version does support only a maximum of 2 "
"top blobs per data layer. Deployment result might be incorrect.".format(
nTooManyTopBlobs
), self._logId)
def _removeDataLayers(self):
"""Remove all data layers from the deployed Network."""
for id, layer in self._dataLayers.iteritems():
self._dHelper.removeLayer(id)
def _removeLayersUsingLabels(self):
"""Remove all layers that require labels as an input."""
# iterate over the original(!) net, but delete from the deployment(!) net
for id, layer in self._originalNetworkDictionary["layers"].iteritems():
if self._isLayerRequiringLabels(layer):
self._dHelper.removeLayer(id)
def _isLayerRequiringLabels(self, layer):
"""Return true, iff the given layer requires any labels as an input."""
if "bottom" in layer["parameters"]:
for bottomBlob in layer["parameters"]["bottom"]:
if bottomBlob in self._labelBlobNames:
return True
return False
def _getLabelBlobName(self, layer):
"""Get the name of the layer's top blob that represents any labels.
Require: layer must be a data layer.
According to http://caffe.berkeleyvision.org/tutorial/data.html, the label blob is always called "label".
Note: This method might need to be adjusted to be able handling further special cases.
"""
if "top" in layer["parameters"] and "label" in layer["parameters"]["top"]:
return "label"
else:
return None
def _getDataBlobName(self, layer):
"""Get the name of the layer's top blob, that represents any input data.
Require: layer must be a data layer.
According to http://caffe.berkeleyvision.org/tutorial/data.html, the data blob is always called "data". However,
the following implementation allows handling (some) special cases by changing only the implementation of
_getLabelBlobName().
"""
labelBlobName = self._getLabelBlobName(layer)
if "top" in layer["parameters"]:
for topBlobName in layer["parameters"]["top"]:
if topBlobName != labelBlobName:
return topBlobName
return None
def _insertInputLayers(self):
"""Insert new input layers with fixed dimensions.
Note that, the number of newly-added input layer might be lower than the number of previously-removed data
layers. On the one hand, we will add only one new input layer for each unique data blob name, while multiple
data layers might have used the same data blob name. On the other hand, input layers will only be added, if
at least one other layer is using the provided data.
"""
inputLayerType = info.CaffeMetaInformation().availableLayerTypes()["Input"]
inputLayerNr = 1
for dataBlobName in self._dataBlobNames:
# create a new input layer with default values and add it to the deployment net
name = self._getNewInputLayerName(inputLayerNr)
inputLayer, inputLayerId = self._dHelper.addLayer(inputLayerType, name, inputLayerNr-1)
# modify the layer template
inputLayer["parameters"]["top"] = [dataBlobName]
# set input_param.shape with batch size 1 and the dimensions of the first data element
inputLayer["parameters"]["input_param"] = dict()
inputLayer["parameters"]["input_param"]["shape"] = []
inputLayer["parameters"]["input_param"]["shape"].append(dict())
inputLayer["parameters"]["input_param"]["shape"][0]["dim"] = self._inputBlobShapes[dataBlobName]
inputLayer["parameters"]["input_param"]["shape"][0]["dim"].insert(0, 1)
# prepare next input layer
inputLayerNr += 1
# check whether there is any shape that could not be determined automatically and needs to be set manually
inputShapeWarning = False
for inputShape in self._inputBlobShapes:
if len(inputShape) < 1:
inputShapeWarning = True
break
if inputShapeWarning:
Log.log("At least one input shape could not be determined automatically. Please open the created prototxt "
"and manually fix all input shapes which include only the batch size (1).", self._logId)
def _getNewInputLayerName(self, inputLayerNr):
"""Get the name for a new input layer.
If only one input layer will be added, the name will always be "data". Otherwise, inputLayerNr will be added as
a suffix.
"""
name = "data"
if len(self._dataBlobNames) > 1:
name += str(inputLayerNr)
return name
def _addSoftmax(self):
"""Add a softmax layer to the very end of the net, but only if a SoftmaxWithLoss layer was used before."""
# check whether the net used to contain a SoftmaxWithLoss layer
softmaxWithLossWasUsed = False
for id, layer in self._originalNetworkDictionary["layers"].iteritems():
if layer["type"].name() == "SoftmaxWithLoss":
softmaxWithLossWasUsed = True
break
if softmaxWithLossWasUsed:
# ensure that the remaining deployment net has at least one layer
if len(self._deployedNetworkDictionary["layers"]) > 0:
softmaxLayerType = info.CaffeMetaInformation().availableLayerTypes()["Softmax"]
# do not add another softmax, if the current deployment network already contains one
softmaxAlreadyIncluded = False
for id, layer in self._deployedNetworkDictionary["layers"].iteritems():
if layer["type"].name() == softmaxLayerType.name():
softmaxAlreadyIncluded = True
break
if not softmaxAlreadyIncluded:
# get the very last layer
lastLayerId = self._deployedNetworkDictionary["layerOrder"][-1]
lastLayer = self._deployedNetworkDictionary["layers"][lastLayerId]
# ensure that the determined last layer does have a top blob
if "top" in lastLayer["parameters"] and len(lastLayer["parameters"]["top"]) > 0:
# create new softmax layer with default values and add it to the deployment net
name = "softmax"
position = len(self._deployedNetworkDictionary["layers"])
softmaxLayer, softmaxLayerId = self._dHelper.addLayer(softmaxLayerType, name, position)
# connect the softmax layer with the existing network
softmaxLayer["parameters"]["bottom"] = [lastLayer["parameters"]["top"][0]]
# name the output
softmaxLayer["parameters"]["top"] = ["probabilities"]
else:
Log.log("Could not add Softmax layer as the very last layer of the deployment net does not have any "
"top blobs.",
self._logId)
else:
Log.log("Could not add Softmax layer as the remaining deployment net does not have any layers.",
self._logId)
Barista/backend/barista/hash.py 0000664 ڊ0011117 00000001311 13265130053 016671 0 ustar klemms o0cvpr import hashlib
import os
def hashFile(path):
"""Hashes a file and returns the hashvalue"""
m = hashlib.md5()
try:
file = open(path, "r")
for chunk in iter(lambda: file.read(4096), b""):
m.update(chunk)
return m.hexdigest()
except Exception as e:
print e
exit -1
def hashDir(path):
"""Hashes a directory and returns the hashvalue"""
hash = ""
for root, dirs, files in os.walk(path):
for names in sorted(files):
try:
filepath = os.path.join(root, names)
hash += hashFile(filepath)
except Exception as e:
print e
exit -1
return hash Barista/backend/barista/project.py 0000664 ڊ0011117 00000072542 13265130053 017432 0 ustar klemms o0cvpr import copy
import json
import os
import re
import shutil
import uuid
from datetime import datetime
from PyQt5.Qt import QObject
from PyQt5.QtCore import pyqtSignal
from PyQt5.QtWidgets import QMessageBox
from backend.barista.constraints.permanent.project import ensureProjectDataConstraints
from backend.barista.session.session import Session
from backend.barista.session.client_session import ClientSession
from backend.barista.session.session import State
from backend.barista.session.session_utils import Paths
from backend.networking.protocol import Protocol
from backend.networking.net_util import sendMsgToHost
import backend.caffe.dict_helper as helper
import backend.caffe.loader as loader
import backend.caffe.proto_info as info
import backend.caffe.saver as saver
import backend.barista.caffe_versions as caffeVersions
from backend.barista.session.session_pool import SessionPool
from backend.barista.utils.logger import Log
from backend.barista.utils.logger import LogCaller
from backend.caffe.proto_info import CaffeMetaInformation
def dirIsProject(dirname):
""" Checks if the dir with the name dirname contains a projectfile """
if os.path.exists(_baristaProjectFile(dirname)):
return True
return False
def _baristaProjectFile(directory):
""" Returns the filename of the config-json file in the given directory """
return os.path.join(directory, "barista_project.json")
class Project(QObject, LogCaller):
""" This class is a abstraction for a caffe project.
A project has a directory which contains the following dependencies:
- a settings file with general settings for the project, e.g. the caffe
version
- a layout file with information about the layout of the net in the prototxt_editor
- a caffe solver file
- two versions of the caffe net file
-> original one: a copy of the net state just as it would appear by using
the "export net" feature
-> internal one: a slightly modified version of the original file. Relative
paths are changed to allow executing caffe inside of a session folder,
although the user defines paths relative to the project folder.
- a log directory
- a directory for training sessions containing log files and snapshots
A project directory has the following structure:
/project_name
|---barista_project.json
|---/logs
|---/sessions
| |---/20161118_153000_1
| | |---solver.prototxt (see FILE_NAME_SOLVER)
| | |---net-original.prototxt (see FILE_NAME_NET_ORIGINAL)
| | |---net-internal.prototxt (see FILE_NAME_NET_INTERNAL)
| | |---/logs
| | | |---project_name_1.1.log
| | | |---project_name_1.2.log
| | |---/snapshots
| | | |---project_name_iter_1000.caffemodel
| | | |---project_name_iter_1000.solverstate
| |---/20161118_161000_2
| | |---solver-internal.prototxt (see FILE_NAME_SOLVER)
| | |---net-original.prototxt (see FILE_NAME_NET_ORIGINAL)
| | |---net-internal.prototxt (see FILE_NAME_NET_INTERNAL)
| | |---/logs
| | | |---project_name_2.1.log
| | |---/snapshots
...
"""
# signals
newSession = pyqtSignal(object)
# The deleteSession signal is called from within backend/barista/session/session.py
# in the Session.delete() function.
deleteSession = pyqtSignal(object)
resetSession = pyqtSignal(object)
activeSessionChanged = pyqtSignal(object)
def __init__(self, directory):
super(Project, self).__init__()
# Make sure needed directoies exists
self.caffeVersion = caffeVersions.getDefaultVersion().getName()
self.projectRootDir = directory
self.projectId = None
self.current_sid = None
self.callerId = None
self.__activeSID = None
self.__sessions = {}
self.__inputManagerState = dict()
self.__transform = None
self.sessions = os.path.join(self.projectRootDir, 'sessions')
# self.logs = os.path.join(self.projectRootDir, 'logs')
self.__ensureDirectory(self.sessions)
# self.__ensureDirectory(self.logs)
self.settings = self.__loadDefaultSettings()
self.loadProject(True)
def projectConfigFileName(self):
""" Returns the filename of the config-json file """
return _baristaProjectFile(self.projectRootDir)
def loadProject(self, set_settings=False):
""" Loads the settings of the project. Returns a state-dictionary for
the NetworkManager.
State looks like: {
"projectid": "xyz...",
"caffeVersion": {...},
"inputdb": { .. },
"network": { .. },
"solver": {...},
"selection": [],
"position": {}
}
"""
filename = self.projectConfigFileName()
projectData = None
if not os.path.exists(filename):
self.__loadDefaultSettings()
self.projectId = str(uuid.uuid4())
sessionID = self.createSession(state_dictionary=None)
self.setActiveSID(sessionID)
else:
with open(filename, "r") as file:
res = json.load(file)
if set_settings and ("environment" in res):
self.settings = res["environment"]
if "inputdb" in res:
self.__inputManagerState = res["inputdb"]
if "caffeVersion" in res:
self.caffeVersion = res["caffeVersion"]
caffeVersions.setDefaultVersion(self.caffeVersion)
if "projectdata" in res: # this is for backward compatibility, only
# Fill missing type-instances
allLayers = info.CaffeMetaInformation().availableLayerTypes()
layers = res["projectdata"]["network"]["layers"]
for id in layers:
typename = layers[id]["parameters"]["type"]
layers[id]["type"] = allLayers[typename]
projectData = res["projectdata"]
if "transform" in res:
self.__transform = res["transform"]
if "projectid" in res:
self.projectId = res["projectid"]
else:
self.projectId = str(uuid.uuid4()) # backwards compatibility
self.loadSessions()
if len(self.__sessions) == 0:
Log.error("Could not find a valid Session, creating new empty Session. Maybe the project "
"contains only remote Sessions?", self.getCallerId())
sessionID = self.createSession()
#check which session was last active, if this is not available, take the last Session in SID order, if no
# valid sessions are available, create a new empty session and set this active
if "activeSession" in res:
self.setActiveSID(res["activeSession"])
elif len(self.getValidSIDs())>0:
self.setActiveSID(self.getValidSIDs()[-1])
else: # this branch should be reached only if an old projekt format is opened, where no sessions are stored
sessionID = self.createSession(state_dictionary=projectData)
self.setActiveSID(sessionID)
if projectData:
self.setActiveSessionStateDict(projectData)
if self.getActiveSession().hasStateDict():
projectData = self.getActiveSession().state_dictionary
else:
if projectData:
# this branch should be reached only if an old projekt format is opened,
# where sessions do not have their own state dictionary
self.setActiveSessionStateDict(projectData)
return projectData, self.__inputManagerState, self.__transform
def getInputManagerState(self):
return self.__inputManagerState
def getViewTransform(self):
return self.__transform
def saveProject(self, networkManagerState=None, inputManagerState=None, transform=None):
""" Saves the settings of this project using the given State of the
NetworkManager.abs
State should looks like: {
"projectid": "xyz...",
"network": { .. },
"caffeVersion": {...},
"inputdb": { .. },
"solver": {...},
"selection": [],
"position": {}
}
"""
if inputManagerState:
self.__inputManagerState = copy.deepcopy(inputManagerState)
if transform:
self.__transform = copy.deepcopy(transform)
for session in self.__sessions.itervalues():
session.save()
# Serialize
tosave = {
"projectid": self.projectId,
"caffeVersion": self.caffeVersion,
"activeSession": self.getActiveSID(),
"inputdb": self.__inputManagerState,
"environment": self.settings,
"transform": [self.__transform.m11(), self.__transform.m12(),
self.__transform.m21(), self.__transform.m22(),
self.__transform.dx(), self.__transform.dy()]
}
with open(self.projectConfigFileName(), "w") as file:
json.dump(tosave, file, sort_keys=True, indent=4)
def __loadDefaultSettings(self):
""" Load settings with sensitive defaults.
"""
self.settings = {
'plotter': {
'logFiles': {},
'checkBoxes': {}
}
}
def changeProjectCaffeVersion(self, version):
"""change the project caffe version"""
self.caffeVersion = version
def getCaffeVersion(self):
"""returns the caffe version name, saved in the project file or, if this isn't set, from Barista default settings"""
return self.caffeVersion
def deletePlotterSettings(self, logId):
"""deletes the plotter settings for a given log, e.g. if a session was reset"""
try:
#check if settings for given logId exist:
if "logId" in self.settings["plotter"]["checkBoxes"]:
return
except Exception:
return
res = None
#check if a settings json exists:
if os.path.exists(self.projectConfigFileName()):
with open(self.projectConfigFileName(), "r") as file:
res = json.load(file)
#determine how much of the json is already created and write accordingly:
if res is None:
res ={"environment": {"plotter": {"checkBoxes": {}}}}
elif "environment" not in res:
res["environment"] = {"plotter": {"checkBoxes": {}}}
elif "plotter" not in res["environment"]:
res["environment"]["plotter"] = {"checkBoxes": {}}
elif logId in res["environment"]["plotter"]["checkBoxes"]:
res["environment"]["plotter"]["checkBoxes"].pop(logId)
#save to file
with open(self.projectConfigFileName(), "w") as file:
json.dump(res, file, sort_keys=True, indent=4)
def getCallerId(self):
""" Return the unique caller id for this project
"""
if self.callerId is None:
self.callerId = Log.getCallerId(self.getProjectName())
return self.callerId
def getProjectName(self):
""" Return the name of the project.
"""
return os.path.basename(os.path.normpath(self.projectRootDir))
def getProjectDirectory(self):
""" Return the project directory.
"""
return self.projectRootDir
def getProjectId(self):
""" Return the ProjectId"""
return self.projectId
def buildSolverPrototxt(self):
""" Load the current solver dictionary and return the corresponding
message object.
:return: A solver message object.
"""
res, inp, _ = self.loadProject()
solver = saver.saveSolver(res["solver"])
return solver
def buildNetPrototxt(self, internalVersion=False):
""" Load the current net dictionary and return the corresponding
message object.
:param internalVersion: Iff true, the loaded net will be modified to
the internal version.
:return: A net message object.
"""
currentState = self.getActiveSession().state_dictionary
netDictionary = None
if currentState:
netDictionary = currentState["network"]
else:
res, _, _= self.loadProject()
netDictionary = res["network"]
if internalVersion:
netDictionary = self._modifyNetDictionaryToInternalVersion(copy.deepcopy(netDictionary))
solver = saver.saveNet(netDictionary)
return solver
def _modifyH5TxtFile(self, dir, state=None):
net = None
if state:
net = state["network"]
else:
session = self.getActiveSession()
if session:
state_dict = session.state_dictionary
if state_dict:
if "network" in state_dict:
net = state_dict["network"]
if net:
h = helper.DictHelper(net)
for layerId, layer in net.get("layers", {}).iteritems():
paramKey = "hdf5_data_param.source"
if h.layerParameterIsSet(layerId, paramKey):
paramValue = h.layerParameter(layerId, paramKey)
if paramValue is not None and not os.path.isabs(paramValue):
newFilename = str(layerId) + ".txt"
newFilepath = os.path.join(dir, newFilename)
oldPath = os.path.join(dir, os.path.join(os.pardir,
os.path.join(os.pardir, paramValue)))
if os.path.exists(oldPath):
with open(newFilepath, "w") as f:
lines = [line.rstrip('\n') for line in open(oldPath)]
for line in lines:
if line is not "":
if line[:1] == '.':
line = os.path.join(os.pardir, os.path.join(os.pardir, line))
f.write("\n" + line)
else:
Log.error('Failed to copy hdf5txt file. File does not exists: ' + oldPath, self.getCallerId())
def getActiveSession(self):
sid = self.getActiveSID()
if sid:
return self.getSession(sid)
else:
return None
def getActiveSID(self):
if self.__activeSID:
if len(self.__sessions) > 0:
if self.__activeSID not in self.__sessions:
Log.log("The Active Session is no longer available. The Project seems to be broken. The active Session is set to the highest ID available.", self.callerId)
self.setActiveSID(self.__sessions.keys()[-1])
Log.log("Active Session set to " + str(self.__activeSID), self.callerId)
return self.__activeSID
def getSession(self, SID):
if SID in self.getValidSIDs():
return self.__sessions[SID]
else:
Log.log("Session " + str(SID) + " could not be loaded. Valid IDs are: " +
", ".join([str(i) for i in self.getValidSIDs()]), self.getCallerId())
return None
def setActiveSessionStateDict(self, state):
self.__sessions[self.__activeSID].setStateDict(stateDict=state)
def setActiveSID(self, sid):
validSIDs = self.getValidSIDs()
if sid in validSIDs:
self.__activeSID = sid
self.activeSessionChanged.emit(sid)
else:
Log.error("Could not set active session to " + str(sid) + " valid Session-IDs are: "+
", ".join([str(s) for s in validSIDs]), self.getCallerId())
if not self.__activeSID:
self.__activeSID = validSIDs[-1] if len(validSIDs) > 0 else None
Log.log("Active session set to " + str(self.__activeSID), self.getCallerId())
def getValidSIDs(self):
validSIDs = [session.getSessionId() for session in self.__sessions.itervalues()]
return validSIDs
def getSettings(self):
""" Return the settings for this project.
"""
return self.settings
def getSnapshots(self):
""" Return the directory for snapshots.
"""
return self.snapshots
def getLogs(self):
""" Return the directory for log files.
"""
return self.logs
def getSessions(self):
""" Return a dictionary of finished and running caffe sessions, keyed
by session id.
"""
return self.__sessions
def getSessionsDirectory(self):
""" Return the directory which contains the sessions.
"""
return self.sessions
def createSession(self, state_dictionary=None):
""" Return a new session instance with a new directory
"""
sid = self.getNextSessionId()
session = Session(self, sid=sid, parse_old=False, state_dictionary=state_dictionary)
self.__sessions[sid] = session
# copy HDF5TXT files
# TODO: This should probably be moved to the Session?
self._modifyH5TxtFile(session.getDirectory(), state=state_dictionary)
self.newSession.emit(sid)
return sid
def createRemoteSession(self, remote, state_dictionary=None):
"""use this only to create entirely new sessions. to load existing use the loadRemoteSession command"""
msg = {"key": Protocol.GETCAFFEVERSIONS}
reply = sendMsgToHost(remote[0], remote[1], msg)
if reply:
remoteVersions = reply["versions"]
if len(remoteVersions) <= 0:
msgBox = QMessageBox(QMessageBox.Warning, "Error", "Cannot create remote session on a host witout a caffe-version")
msgBox.addButton("Ok", QMessageBox.NoRole)
msgBox.exec_()
return None
sid = self.getNextSessionId()
msg = {"key": Protocol.CREATESESSION, "pid": self.projectId, "sid": sid}
layers = []
for layer in state_dictionary["network"]["layers"]:
layers.append(state_dictionary["network"]["layers"][layer]["parameters"]["type"])
msg["layers"] = layers
ret = sendMsgToHost(remote[0], remote[1], msg)
if ret:
if ret["status"]:
uid = ret["uid"]
else:
for e in ret["error"]:
Log.error(e, self.getCallerId())
return None
else:
Log.error('Failed to create remote session! No connection to Host', self.getCallerId())
return None
session = ClientSession(self, remote, uid, sid)
if state_dictionary is not None:
session.state_dictionary = state_dictionary
self.__sessions[sid] = session
self.newSession.emit(sid)
return sid
def loadRemoteSession(self, remote, uid):
sid = self.getNextSessionId()
session = ClientSession(self, remote, uid, sid)
availableTypes = info.CaffeMetaInformation().availableLayerTypes()
unknownLayerTypes = []
for layerID in session.state_dictionary["network"]["layers"]:
type = session.state_dictionary["network"]["layers"][layerID]["parameters"]["type"]
if not type in availableTypes and not type in unknownLayerTypes:
unknownLayerTypes.append(type)
if len(unknownLayerTypes) > 0:
msg = "Cannot load session. The selected session contains layers unknown to the current caffe-version.\n\nUnknown layers:"
for type in unknownLayerTypes:
msg += "\n" + type
msgBox = QMessageBox(QMessageBox.Warning, "Warning", msg)
msgBox.addButton("Ok", QMessageBox.NoRole)
msgBox.exec_()
session._disconnect()
return None
self.__sessions[sid] = session
self.newSession.emit(sid)
return sid
def cloneRemoteSession(self, oldSolverstate, oldSession):
"""
Starts the cloning process for a remote session and creates the corr. local session upon success
oldSolverstate: solverstate produced by the snapshot from which the clone should be created
oldSession: session from which a clone should be created (type ClientSession)
"""
# validate the given session and solverstate
if oldSolverstate is None:
Log.error('Could not find solver',
self.getCallerId())
return None
if oldSession is None:
Log.error('Failed to create session!', self.getCallerId())
return None
sid = self.getNextSessionId()
# call the remote host to invoke cloning; @see cloneSession in server_session_manager.py
msg = {"key": Protocol.CLONESESSION, "pid": self.projectId,
"sid": sid, "old_uid": oldSession.uid, "old_solverstate": oldSolverstate}
ret = sendMsgToHost(oldSession.remote[0], oldSession.remote[1], msg)
# receive and validate answer
if ret:
if ret["status"]:
uid = ret["uid"]
else:
for e in ret["error"]:
Log.error(e, self.getCallerId())
return None
else:
Log.error('Failed to clone remote session! No connection to Host', self.getCallerId())
return None
# Create a corr. local session and copy (if available) the state-dictionary to maintain
# solver/net etc.
session = ClientSession(self, oldSession.remote, uid, sid)
if hasattr(oldSession, 'state_dictionary'):
session.state_dictionary = oldSession.state_dictionary
self.__sessions[sid] = session
self.newSession.emit(sid)
return sid
def cloneSession(self, oldSolverstate, oldSession):
"""
Creates a new session with the same net/solver etc. and a additional .caffemodel file with pretrained weights
oldSolverstate: solverstate produced by the snapshot from which the clone should be created
oldSession: session from which a clone should be created (type ClientSession)
"""
if type(oldSolverstate) is not str and type(oldSolverstate) is not unicode:
# if no valid solverstate is given, take the last model created for this session
oldCaffemodel = oldSession.getLastModel()
else:
snapshotDir = os.path.join(oldSession.getSnapshotDirectory(), oldSolverstate)
oldCaffemodel = loader.getCaffemodelFromSolverstate(snapshotDir)
if oldCaffemodel is None:
Log.error('Could not find model',
self.getCallerId())
return None
if oldSession:
oldSnapshotDir = oldSession.getSnapshotDirectory()
if os.path.isabs(oldSnapshotDir):
# locate caffemodel
oldModelPath = os.path.join(oldSnapshotDir, oldCaffemodel)
else:
oldSessionDir = oldSession.getDirectory()
oldModelPath = os.path.join(oldSessionDir, oldSnapshotDir, oldCaffemodel)
else:
Log.error('Failed to create session!', self.getCallerId())
return None
# create new session
sessionID = self.createSession()
if sessionID is None:
Log.error('Failed to create session!', self.getCallerId())
return None
self.__sessions[sessionID].setParserInitialized()
# create directories for the new session
newSnapshotDir = self.__sessions[sessionID].getSnapshotDirectory()
self.__ensureDirectory(self.__sessions[sessionID].getDirectory())
self.__ensureDirectory(newSnapshotDir)
if os.path.isdir(newSnapshotDir):
newCaffemodel = 'pretrained.caffemodel'
newModelPath = os.path.join(newSnapshotDir, newCaffemodel)
try:
# copy the old caffemodel to the new location
shutil.copy2(oldModelPath, newModelPath)
# initialize new session
self.__sessions[sessionID].setPretrainedWeights(newCaffemodel)
self.__sessions[sessionID].iteration = 0
self.__sessions[sessionID].max_iter = oldSession.max_iter
except Exception as e:
Log.error('Failed to copy caffemodel to new session: '+str(e),
self.getCallerId())
# copy the old state-dict into the new session
self.__sessions[sessionID].setStateDict(oldSession.state_dictionary)
self.__sessions[sessionID].setState(State.WAITING)
self.__sessions[sessionID].save(includeProtoTxt=True)
return sessionID
else:
self.__sessions[sessionID].delete()
Log.error('Snapshot directory '+newSnapshotDir+' does not exist!',
self.getCallerId())
return None
def rebuildSession(self, directory):
""" Rebuild the session from the given directory.
Return: The session if the directory is a valid session directory.
"""
session = Session(self, directory=directory, parse_old=False)
return session
def loadSessions(self):
""" Load sessions which are found in the sessions directory.
Create a session object for every session and key it by the session id.
"""
pool = SessionPool()
sdir = self.getSessionsDirectory()
count = 0
for entry in os.listdir(sdir):
sess_dir = os.path.join(sdir, entry)
if self.isSession(sess_dir):
session = self.rebuildSession(sess_dir)
if session:
self.__sessions[session.getSessionId()] = session
pool.addSession(session)
self.newSession.emit(session.getSessionId())
count += 1
return count
def removeSessions(self, sessions):
""" Remove the sessions from the project.
"""
for session in sessions:
sid = session.getSessionId()
session.delete()
self.__sessions.pop(sid)
del session
def closeSession(self, remoteSession):
""" Close a remote session and remove it from the available sessions"""
sid = remoteSession.getSessionId()
remoteSession.close()
self.__sessions.pop(sid)
del remoteSession
def isSession(self, directory):
""" Checks if the directory contains a valid session."""
if not os.path.exists(directory):
Log.error('Session directory '+directory+' does not exist', self.getCallerId())
return False
session_file = os.path.join(directory, Paths.FILE_NAME_SESSION_JSON)
if not os.path.exists(session_file):
Log.error('Session directory '+directory+' does not contain a session file.', self.getCallerId())
return False
with open(session_file, 'r') as file:
d = json.load(file)
jsonKeys = ("Iteration", "MaxIter", "NetworkState", "ProjectID", "SessionState")
keyNotFound=False
for key in jsonKeys:
if key not in d:
Log.error("Session directory "
+ os.path.basename(os.path.normpath(dir)) + " is invalid! Key '"
+ key + "' is missing in sessionstate!", self.getCallerId())
keyNotFound = True
if keyNotFound:
Log.error("Session at %s is invalid. Key %s in 'sessionstate.json' is missing", self.getCallerId())
return False
return True
def hasRunningSessions(self):
""" Return True if there are running sessions in this project.
"""
for sid, session in self.__sessions.iteritems():
if session.getState() is State.RUNNING:
return True
return False
def pauseRunningSessions(self):
""" Pause all running sessions.
"""
for sid, session in self.__sessions.iteritems():
if session.getState() is State.RUNNING:
session.pause()
def getNextSessionId(self):
""" Return the next session id from the sequence of ids.
"""
sIDs = [s for s in self.__sessions.iterkeys()]
if len(sIDs) > 0:
return max(max(sIDs)+1, 1)
else:
return 1
def __ensureDirectory(self, directory):
""" Creates a directory if it does not exist.
"""
if directory == '':
return
if not os.path.exists(directory):
try:
os.makedirs(directory)
Log.log('Created directory: '+directory, self.getCallerId())
except Exception as e:
Log.error('Failed to create directory '+directory+')'+str(e),
self.getCallerId())
def parseSnapshotPrefixFromFile(self, filename):
""" Return the snapshot prefix of the solver.
"""
with open(filename) as f:
regex_prefix = re.compile(
'snapshot_prefix:[\s]+"(.+)"')
for line in f:
prefix_match = regex_prefix.search(line)
if prefix_match:
return prefix_match.group(1) Barista/backend/caffe/__init__.py 0000664 ڊ0011117 00000000000 13265130053 017116 0 ustar klemms o0cvpr Barista/backend/caffe/check_hardware.py 0000664 ڊ0011117 00000005364 13265130053 020333 0 ustar klemms o0cvpr from subprocess import Popen, PIPE, STDOUT
from sys import stdout
from os import path
import logging
from backend.networking.protocol import Protocol
def checkHardware(binary, silent=False, transaction=None):
"""
probe caffe continuously for incrementing until missing id
structure:
[
{ "id": 0,
"name": "..",
"log": ["..", "..", "..", ... ]
},
{ "id": 1,
"name": "..",
"log": ["..", "..", "..", ... ]
},
...
]
"""
gid = 0
hw = []
if not silent:
stdout.write("Checking Hardware...\n")
logging.info("Checking Hardware...")
cpu = _getCPU()
name = _getCPUName(cpu)
hw.append({"name": name, "log": cpu})
if not silent:
stdout.write("CPU found: " + name + "\n")
logging.info("CPU found: %s", name)
if transaction:
msg = {"key": Protocol.SCANHARDWARE, "finished": False, "name": name}
transaction.send(msg)
while True:
log = _getId(gid, binary)
if not _isValid(log) or _isCpuOnly(log):
if not silent and gid is 0:
stdout.write("No GPU found, CPU mode\n")
logging.info("No GPU found, CPU mode")
break
name = _getName(log)
if not silent:
stdout.write("GPU " + str(gid) + " found: " + name + "\n")
if transaction:
msg = {"key": Protocol.SCANHARDWARE, "finished": False, "name": name, "id": gid}
transaction.send(msg)
hw.append({"id": gid, "name": name, "log": _parseLog(log)})
gid += 1
return hw
def _getId(gid, binary):
"""probe caffe for gpu id"""
proc = Popen([binary, "device_query", "-gpu", str(gid)], stdout=PIPE, stderr=STDOUT)
it = iter(proc.stdout.readline, '')
log = []
for i in it:
log.append(i[:-1])
return log
def _getCPU():
proc = Popen(["cat", "/proc/cpuinfo"], stdout=PIPE)
it = iter(proc.stdout.readline, '')
log = []
for i in it:
log.append(i[:-1])
return log
def _isValid(log):
"""check if no error has occurred"""
for l in log:
if "error" in l.lower():
return False
return True
def _isCpuOnly(log):
"""check for CPU-Only mode"""
for l in log:
if "cpu" in l.lower():
return True
return False
def _getName(log):
"""extract the Name"""
for l in log:
if "name:" in l.lower():
return l[l.index("Name:") + 5:].strip()
return ""
def _getCPUName(log):
"""extract the CPU name"""
for l in log:
if "model name" in l.lower():
return l.split(":")[1].strip()
def _parseLog(log):
"""clean up logs to show to user"""
return [l[l.index("]")+2:] for l in log[1:]]
Barista/backend/caffe/dict_helper.py 0000664 ڊ0011117 00000026057 13265130053 017665 0 ustar klemms o0cvpr from backend.caffe.loader import _extract_param
import backend.caffe.proto_info as info
import uuid
import copy
from google.protobuf.internal.containers import RepeatedScalarFieldContainer as ProtoList
def isNetDictValid(netdic):
""" This function returns a tuple (valid, msg). Value is true iff netdic has a valid format.
If the format is not valid, msg contains a text which describes the problem.
"""
if not {"layers", "layerOrder"}.issubset(netdic.keys()):
missing = {"layers","layerOrder"}.difference(netdic.keys())
return False, "Needed parameters {} not in dictionary".format(list(missing))
layers = netdic["layers"]
order = netdic["layerOrder"]
#Check if netparameters are valid (except layers)
netparam = info.CaffeMetaInformation().availableNetParameters().copy()
del netparam["layers"]
tmp = netdic.copy()
del tmp["layers"]
del tmp["layerOrder"]
if not _areParameterValid(netparam, tmp):
return False, "Invalid net paremters"
# Check for correct types of this special values
if type(order) != list or type(layers) != dict:
return False, "layerOrder or layers are invalid types"
# Test if all ids are in the layerOrder list and vise versa
if set(layers.keys()) != set(order):
missingOrder = set(layers.keys()).difference(order)
missingLayer = set(order).difference(layers.keys())
return False, "layerOrder and layers does not have same ids - missing ids in layers: {} ; missing ids in order: {}"\
.format(missingLayer, missingOrder)
# Check every sublayer
for id in order:
if not {"parameters","type"}.issubset(layers[id].keys()):
return False, "parameters or type are not in layers subdictionary"
params = layers[id]["parameters"]
if params is None:
return False, "parameters is None, should be at least {'name': '..', 'type': '...'}"
layer = layers[id]["type"]
if not isinstance(layer, info.LayerType):
return False, "layers is {}, expected instance of protoinfo.LayerType".format(layer)
validparam, msg = _areParameterValid(layer.parameters(), params)
if not validparam:
return False, "Invalid parameters for {}: {}".format(id, msg)
return True, ""
def _areParameterValid(allparams, paramdict, prefix=""):
""" Checks if all parameters in paramdict are valid.
allparams should contain all available parameters names as key
and the proto-meta-info-class as value.
"""
def checkType(meta, value, typeprefix):
pytype = None
if meta.isParameterGroup():
if type(value) != dict:
return False, "Expected dictionary , found {}".format(type(value))
return _areParameterValid(meta.parameter(), value, typeprefix)
elif meta.isBool():
pytype = [bool]
elif meta.isString():
pytype = [unicode, str]
elif meta.isInt():
pytype = [int, long]
elif meta.isBytes():
pass # TODO: What type are bytes?
elif meta.isFloat() or meta.isDouble():
pytype = [float]
elif meta.isEnum():
pytype = [str, unicode]
res = type(value) in pytype
if not res:
return False, "Expected Type {}, found {}".format(pytype, type(value))
return True, ""
for param in paramdict:
typeprefix=prefix+"."+param
val = paramdict[param]
if not allparams.has_key(param):
return False, "Parameter {}: Parameter not available in Caffe".format(typeprefix)
meta = allparams[param]
if meta.isRepeated():
if not (type(val) is ProtoList or type(val) is list):
return False, "Parameter {}: Expected repeated entry, found single one".format(typeprefix)
for i,entry in enumerate(val):
valid, msg = checkType(meta,entry, typeprefix)
if not valid:
return False, "Parameter {} - idx {}: {}".format(typeprefix, i,msg)
else:
valid, msg = checkType(meta,val,typeprefix)
if not valid:
return False, "Parameter {}: {}".format(typeprefix,msg)
return True, ""
def bareNet(name):
""" Creates a dictionary of a networks with default values where required. """
from backend.caffe.path_loader import PathLoader
proto = PathLoader().importProto()
net = proto.NetParameter()
descr = info.ParameterGroupDescriptor(net)
params = descr.parameter().copy()
del params["layer"]
del params["layers"]
res = _extract_param(net, params)
res["layers"] = {}
res["layerOrder"] = []
res["name"] = unicode(name)
return res
def _bareLayer(layertype, name):
""" Creates a dictionary of the given layertype with the given name
initialized with default values if required.
"""
from backend.caffe.path_loader import PathLoader
proto = PathLoader().importProto()
res = {"type": layertype}
layerParamInst = proto.LayerParameter()
res["parameters"] = _extract_param(layerParamInst, info.CaffeMetaInformation().commonParameters())
res["parameters"]["name"] = unicode(name)
layerparamkeys = layertype.layerParamKeys()
for layerparamkey in layerparamkeys:
layerTypeParam = layertype.parameters()[layerparamkey]
layerTypeParamInst = layerTypeParam.protoclass()()
res["parameters"][layerparamkey] = _extract_param(layerTypeParamInst, layerTypeParam.parameter())
res["parameters"]["type"] = unicode(layertype.name())
return res
class DictHelper:
""" A helper class to simplify the using of a network dictionary """
def __init__(self,netdic):
self._netdic = netdic
def nameOfLayers(self):
""" Return a list with all layers of the networks """
return [x["parameters"]["name"] for x in self._netdic["layers"].values()]
def nameOfTopBlobs(self):
""" Return a list with all top blob names in use of the current network """
names = []
for id in self._netdic["layers"]:
if "top" in self.layerPerId(id)["parameters"]:
for j in range(0, len(self.layerPerId(id)["parameters"]["top"])):
names.append(self.layerPerId(id)["parameters"]["top"][j])
return names
def netParameters(self):
""" Return all parameters for the network (excluding layers) """
return dict([(k,self._netdic[k]) for k in self._netdic if not k in ["layers","layerOrder"]])
def layerIdsForName(self,name):
""" Return a list of the ids for every layer with the given name """
return [x for x in self._netdic["layers"] if self._netdic["layers"][x]["parameters"]["name"] == name]
def layerType(self,id):
""" Return the type of the layer with the given id
"""
return self._netdic["layers"][id]["type"]
def layerParams(self,id):
""" Return the parameter of the layer with the given id
"""
return self._netdic["layers"][id]["parameters"]
def layerPerId(self, id):
""" Return the layer with the given id """
return self._netdic["layers"][id]
def hasLayerWithId(self,id):
""" Return true if there is a layer with the given id"""
return id in self._netdic["layers"]
def addLayer(self, layertype, name, idx):
""" Add a new Layer from layertype with the given name
and initialize it with the default values if required.
This new layer is added to the intern dictionary at position idx
and the layer subdictionary will be returned with the id.
import backend.caffe.protoinfo as info
convLayer = info.CaffeMetaInformation().availableLayerTypes()["Convolution"]
newLayer, layerId = dichelperinstance.addLayer(convLayer, "Convolution", 3)
"""
newLayer = _bareLayer(layertype, name)
id = str(uuid.uuid4())
self._netdic["layerOrder"].insert(idx, id)
self._netdic["layers"][id] = newLayer
return newLayer, id
def removeLayer(self, id):
del self._netdic["layers"][id]
self._netdic["layerOrder"].remove(id)
def duplicateLayer(self, id, newName, idx=None):
""" Duplicate Layer with given id. The new layer has a new random id
and the new given name.
The layer will be added at position idx in the layerOrder.
If idx==None, the idx will be directly beside the old layer.
This function returns the new layer as dictionary and its id.
E.g. newLayerDict, newId = dicthelper.duplicateLayer("somerandomid", "Some awesome name")
"""
oldLayer = self.layerPerId(id)
newLayer = {
"type": oldLayer["type"],
"parameters": copy.deepcopy(oldLayer["parameters"])
}
newLayer["parameters"]["name"] = newName
newId = str(uuid.uuid4())
if idx is None:
idx = self._netdic["layerOrder"].index(id)+1
self._netdic["layerOrder"].insert(idx, newId)
self._netdic["layers"][newId] = newLayer
return newLayer, newId
def dictionary(self):
return self._netdic
def layerParameterIsSet(self, layerId, parameterKey):
""" Check whether the given parameter is set for the given layer.
:param layerId: The id of the layer to check.
:param parameterKey: The key of the parameter to check. Might represent a nested parameter concatenated with dots.
:return:
"""
isSet = "parameters" in self._netdic["layers"][layerId]
if isSet:
paramKeyParts = parameterKey.split(".")
value = self._netdic["layers"][layerId]["parameters"]
for paramKeyPart in paramKeyParts:
if paramKeyPart in value:
value = value[paramKeyPart]
else:
isSet = False
break
return isSet
def layerParameter(self, layerId, parameterKey):
""" Get a specific parameter for one of the layers.
:param layerId: The id of the layer.
:param parameterKey: The key of the requested parameter. Might represent a nested parameter concatenated with dots.
:return:
"""
paramKeyParts = parameterKey.split(".")
value = self._netdic["layers"][layerId]["parameters"]
for paramKeyPart in paramKeyParts:
if paramKeyPart in value:
value = value[paramKeyPart]
else:
value = None
break
return value
def setLayerParameter(self, layerId, parameterKey, value):
""" Set a specific parameter for one of the layers.
:param layerId: The id of the layer.
:param parameterKey: The key of the requested parameter. Might represent a nested parameter concatenated with dots.
:return:
"""
paramKeyParts = parameterKey.split(".")
dic = self._netdic["layers"][layerId]["parameters"]
for key in paramKeyParts[:-1]:
dic = dic.setdefault(key, {})
dic[paramKeyParts[-1]] = value
Barista/backend/caffe/loader.py 0000664 ڊ0011117 00000017421 13265130053 016644 0 ustar klemms o0cvpr import uuid
import h5py as h5
from google.protobuf import text_format
from google.protobuf.text_format import ParseError
import backend.caffe.proto_info as info
from backend.barista.utils.logger import Log
class ParseException(Exception):
def __init__(self, msg):
self._msg = msg
def __str__(self):
return self._msg
def loadSolver(solverstring):
""" Return a dictionary which represent the caffe-solver-prototxt solverstring """
from backend.caffe.path_loader import PathLoader
proto = PathLoader().importProto()
solver = proto.SolverParameter()
# Get DESCRIPTION for meta infos
descr = info.ParameterGroupDescriptor(solver)
# "Parse" the solver-definition in prototxt-format
try:
text_format.Merge(solverstring,solver)
except ParseError as ex:
raise ParseException(str(ex))
params = descr.parameter().copy() # All Parameters of the solver
return copy.deepcopy(_extract_param(solver,params))
import copy
def loadNet(netstring):
""" Load the prototxt string "netstring" into a dictionary.
The dictionary has the following form
{
"name": "Somenetwork",
"input_dim": [1,2,1,1],
"state": {
"phase": "TRAIN"
},
...
"layers":
{
"somerandomid1": {
"type": LayerType Instance of Pooling-Layer,
"parameters": {
"pooling_param": [
"kernel_size": 23,
"engine": "DEFAULT"
]
....
"input_param": [
{"shape": {"dim": [...], .... },
{"shape": {"dim": [...], .... },
]
}
},
"somerandomid2": {"type": ..., "parameters": ....}
},
"layerOrder": ["somerandomid1", "somerandomid2", ....]
}
"""
from backend.caffe.path_loader import PathLoader
proto = PathLoader().importProto()
# Load Protoclass for parsing
net = proto.NetParameter()
# Get DESCRIPTION for meta infos
descr = info.ParameterGroupDescriptor(net)
# "Parse" the netdefinition in prototxt-format
try:
text_format.Merge(netstring,net)
except ParseError as ex:
raise ParseException(str(ex))
params = descr.parameter().copy() # All Parameters of the network
# add logger output if deprecated layers have been found, to inform the user that those can't be parsed yet
if len(net.layers) > 0:
callerId = Log.getCallerId('protoxt-parser')
Log.log("The given network contains deprecated layer definitions which are not supported and will be dropped.",
callerId)
# Layers is deprecated, Layer will be handled seperatly and linked to "Layers" key
del params["layers"]
del params["layer"]
if params.has_key("layerOrder"):
raise ValueError('Key layerOrder not expected!')
# Extract every other parameters
res = _extract_param(net,params)
res["layers"], res["layerOrder"] = _load_layers(net.layer)
res = copy.deepcopy(res)
return res
def _load_layers(layerlist):
""" Build the dictionary of all layers in layerlist. The dictionary has the form loadNet needs.
"""
allLayers = info.CaffeMetaInformation().availableLayerTypes()
order = []
res = {}
dicLayerTypeCounter = {}
for layer in allLayers:
dicLayerTypeCounter[layer] = 1
for layer in layerlist:
typename = layer.type
layerinfo = info.CaffeMetaInformation().getLayerType(typename)
id = str(uuid.uuid4())
res[id]={
"type": layerinfo,
"parameters": _extract_param(layer,layerinfo.parameters())
}
order.append(id)
for id in order:
if "name" not in res[id]["parameters"]:
typeName = res[id]["parameters"]["type"]
newName = typeName + " #" + str(dicLayerTypeCounter[typeName])
changed = True
while changed == True:
newName = typeName + " #" + str(dicLayerTypeCounter[typeName])
changed = False
for id2 in order:
if "name" in res[id2]["parameters"] and res[id2]["parameters"]["name"] == newName:
dicLayerTypeCounter[typeName] += 1
changed = True
res[id]["parameters"]["name"] = newName
return res, order
def _extract_param(value,parameters):
""" Build the dictionary of all paramters, e.g. for layer. The parameters will be constructed recursivly.
"value" is the loaded class from caffe.proto.caffe_pb2 and "parameters" the dictionary of
parameter-descriptions (wrapped with protoinfo.py-Classes).
"""
res = {}
for paramname in parameters:
val = getattr(value, paramname)
parameter = parameters[paramname]
if parameter.isRepeated():
if len(val) == 0: # probably not set
continue
if parameter.isParameterGroup():
res[paramname] = [_extract_param(subval, parameter.parameter()) for subval in val]
elif parameter.isEnum():
res[paramname] = [copy.deepcopy(parameter.availableValues()[x]) for x in val]
else:
res[paramname] = list(val)
else:
if not value.HasField(paramname):
#Not explicit set -> ignore
continue
if parameter.isParameterGroup():
res[paramname] = _extract_param(val, parameter.parameter())
elif parameter.isEnum():
res[paramname] = copy.deepcopy(parameter.availableValues()[val])
else:
res[paramname] = val
return res
def extractNetFromSolver(solverstring):
"""Read a protoxt string(!) of a solver and return the network protoxt string(!).
This works only, if the solver specifies a network using the "net_param" parameter. A reference to a file using the
"net" parameter can not be handled by this method.
"""
from backend.caffe.path_loader import PathLoader
proto = PathLoader().importProto()
# create empty solver message
solver = proto.SolverParameter()
# "Parse" the solver-definition in prototxt-format
try:
text_format.Merge(solverstring, solver)
# extract net as a message and convert it into a string
netString = text_format.MessageToString(solver.net_param)
return netString
except ParseError as ex:
raise ParseException(str(ex))
def getCaffemodelFromSolverstate(solverstate):
""" Parse the filename of the caffemodel file from the solverstate file.
"""
from backend.caffe.path_loader import PathLoader
proto = PathLoader().importProto()
try:
state = proto.SolverState()
with open(solverstate, 'rb') as f:
state.ParseFromString(f.read())
return state.learned_net
except Exception as e:
print(str(e))
def getCaffemodelFromSolverstateHdf5(filename):
""" Extract the filename of the caffemodel file from the solverstate hdf5 file. """
try:
file = h5.File(filename, 'r')
if "learned_net" in file.keys():
return str(file["learned_net"].value)
return None
except:
return None
def getIterFromSolverstate(solverstate):
""" Parse the iterations from the solverstate file.
"""
from backend.caffe.path_loader import PathLoader
proto = PathLoader().importProto()
try:
state = proto.SolverState()
with open(solverstate, 'rb') as f:
state.ParseFromString(f.read())
return state.iter
except Exception as e:
print(str(e))
Barista/backend/caffe/path_loader.py 0000664 ڊ0011117 00000002677 13265130053 017667 0 ustar klemms o0cvpr import imp
import backend.barista.caffe_versions as caffeVersions
import os
class Singleton(type):
"""This metaclass is used to provide the singleton pattern in a generic way.
See http://stackoverflow.com/a/6798042 for source and further explanation.
TODO outsource this class to use the same pattern in the complete project?
"""
_instances = {}
def __call__(cls, *args, **kwargs):
if cls not in cls._instances:
cls._instances[cls] = super(Singleton, cls).__call__(*args, **kwargs)
return cls._instances[cls]
class PathLoader:
__metaclass__ = Singleton
def __init__(self):
self.path = caffeVersions.getDefaultVersion().getPythonpath()
def importCaffe(self):
import importlib
import sys
sys.path.insert(0, self.path) # stellt sicher, dass der angegebene Pfad als erstes durchsucht wird
try:
caffe = importlib.import_module("caffe")
sys.path.pop(0)
return caffe
except ImportError as e:
print(e)
exit(1)
def importProto(self):
import importlib
import sys
sys.path.insert(0, self.path) # stellt sicher, dass der angegebene Pfad als erstes durchsucht wird
try:
proto = importlib.import_module("caffe.proto.caffe_pb2")
sys.path.pop(0)
return proto
except ImportError as e:
print(e)
exit(1) Barista/backend/caffe/proto_description.py 0000664 ڊ0011117 00000023432 13265130053 021143 0 ustar klemms o0cvpr import os
import re
from backend.barista.utils.settings import applicationQSetting
from backend.barista.utils.logger import Log
class Singleton(type):
"""This metaclass is used to provide the singleton pattern in a generic way.
See http://stackoverflow.com/a/6798042 for source and further explanation.
TODO outsource this class to use the same pattern in the complete project?
"""
_instances = {}
def __call__(cls, *args, **kwargs):
if cls not in cls._instances:
cls._instances[cls] = super(Singleton, cls).__call__(*args, **kwargs)
return cls._instances[cls]
class CaffeProtoParser:
"""This class parses the caffe.proto file.
This is different from what is done in other files of this package, as this is the first time we really parse the
text file itself on our own. Other methods rely on meta data provided after(!) parsing/compiling that file.
"""
__metaclass__ = Singleton
def __init__(self, replaceLineBreaks=True):
"""
replaceLineBreaks:
If True, all line breaks will be replaced by spaces. This is the default behavior as most line breaks are only
caused by the limited code width rather than for semantic reasons.
Anyway, there are also some parameter descriptions using linebreaks on purpose (e.g. lr_policy).
(This does not affect cases in which there is an empty comment line. The latter will be kept anyway.)
"""
self._replaceLineBreaks = replaceLineBreaks
self._fileContent = self._readFile()
self._fieldDescriptions, self._messageDescriptions = self._parseComments()
def _readFile(self):
"""Read the content of the caffe.proto file."""
import backend.barista.caffe_versions as caffeVersions
# get path to caffe.proto file
caffeProtoFilePath = caffeVersions.getAvailableVersions()[0].getProtopath()
if os.path.isfile(caffeProtoFilePath):
# open caffe.proto file
with open(caffeProtoFilePath) as f:
# read complete file at once
content = f.read()
else:
callerId = Log.getCallerId("parameter_descriptions")
Log.error("ERROR: Unable to load caffe.proto file. Parameter descriptions and deprecated flags "
"will be left empty. Did you configure the environment variable CAFFE_ROOT?", callerId)
content = ""
return content
def _parseComments(self):
"""Parse all parameter descriptions provided in the C-style comments.
Results will be provided by the fieldDescriptions() and messageDescriptions() methods.
"""
# our result cache
fieldDescr = {}
msgDescr = {}
# get all defined messages and their content to be able grouping their parameters
# regex inside of outer msg brackets:
# repeat (any chars except brackets) or (opening bracket + any chars except brackets + closing brackets)
msgBlocks = re.findall('((//[^\n]*\n)*)message\s*(\w+)\s*\{(' # begin of message definition
'([^\{\}]|(\{[^\{\}]*\}))*'
')\}', self._fileContent) # end of message definition
for block in msgBlocks:
# only some of the groups are relevant. ignore the rest.
msgName = block[2]
msgContent = block[3]
# first of all: get the comment right above the message itself. not the ones included.
msgCommentLines = block[0].splitlines()
msgComment = ""
for line in msgCommentLines:
msgComment = self._proccessCommentLine(line, msgComment)
if len(msgComment) > 0:
msgDescr[msgName] = self._finalizeComment(msgComment)
# collect all parameters of this message, before adding them to the global result
msgParams = {}
# parse line by line of the message's content and add it to either a comment, parameter, or nothing
msgLines = msgContent.splitlines()
currComment = ""
for line in msgLines:
# check whether this is a comment line or a parameter line
# (if a comment has been added next to a param definition, it might be both)
isComment = re.search(r'.*//\s*(.*)\s*', line) is not None # do not restrict to leading whitespaces!
nonEnumParamMatch = re.search(r'\s*(repeated|optional|required)\s*(\w+)\s*(\w+)\s*=', line)
enumParamMatch = re.search(r'\s*enum\s*(\w+)\s*\{', line)
isParam = nonEnumParamMatch is not None or enumParamMatch is not None
# if this is a comment line..
if isComment:
# add line to current comment
currComment = self._proccessCommentLine(line, currComment)
# if the last lines have build a comment and this line contains a new parameter definition..
if len(currComment) > 0 and isParam:
# add the current parameter to the list of the current message
if nonEnumParamMatch is not None:
paramName = nonEnumParamMatch.group(3)
else:
paramName = enumParamMatch.group(1)
msgParams[paramName] = self._finalizeComment(currComment)
# reset current comment, if this line isn't one anymore or if it is a inline-comment that belongs
# to the parameter in the same line => this param was already processed
if not isComment or isParam:
currComment = ""
# if any parameter descriptions have been found for this message, add it to the global result
if len(msgParams) > 0:
fieldDescr[msgName] = msgParams
return fieldDescr, msgDescr
def _proccessCommentLine(self, line, currComment):
"""Process line as a comment line and append the result to currComment."""
# the content of this line without everything which comes before the comment slashes
commentReducible = re.search(r'.*//\s*(.*)\s*', line)
if commentReducible is not None:
lineContent = commentReducible.group(1)
else: # might be empty or reduced already
lineContent = line
# check whether this comment line does only contain whitespaces
isNotEmpty = re.search(r'\S+', lineContent)
# recreate linebreaks or replace them with spaces
# (don't add anything at the beginning or if the last char is a newline, which might happen in all configs)
if len(currComment) > 0 and currComment[-1] != "\n":
if isNotEmpty is not None:
if self._replaceLineBreaks:
currComment += " "
else:
currComment += "\n"
else:
# if this is an empty comment line, add an empty line to the full comment, too.
currComment += "\n\n"
# add text of current line to current comment
if isNotEmpty is not None:
currComment += lineContent
return currComment
def _finalizeComment(self, comment):
"""Call this method for each comment after each line has been processed by _proccessCommentLine()."""
# remove multiple spaces following each other
while " " in comment:
comment = comment.replace(" ", " ")
# unify descriptions: add a "." at the end of each non-empty comment.
if len(comment) > 0 and comment[-1] != ".":
comment += "."
# capitalize first letter
comment = comment.capitalize()
return comment
def fieldDescriptions(self):
"""Get all descriptions provided in the caffe.proto file's comments, that belong to a field inside of a message.
Returned value is a dictionary of dictionaries. Keys of the first dimension are names of messages,
e.g. 'AccuracyParameter'. Keys of the second dimension are parameter names of those messages, such as 'top_k'.
Finally, each of those keys is connected to a string containing the actual description. If a message/parameter
does not have any descriptions, there won't be a dictionary entry for that element at all.
"""
return self._fieldDescriptions
def messageDescriptions(self):
"""Get all descriptions provided in the caffe.proto file's comments, that belong to a message itself.
The returned value is a dictionary. Keys are message names and values their descriptions as strings.
"""
return self._messageDescriptions
def description(self, msgName, fieldName=None, fieldMsgDefault=None):
"""Get a single description for either a field of a message or a message itself.
fieldName: If None, the message description for msgName will be returned.
Otherwise, the description for fieldName of msgName.
fieldMsgDefault: If field is also a message, the general message description of
messageDescriptions()[fieldMsgDefault] will be used as a default value for field.
"""
description = ""
if fieldName is None: # message description
if msgName in self.messageDescriptions():
description = self.messageDescriptions()[msgName]
else: # field description
if msgName in self.fieldDescriptions() and fieldName in self.fieldDescriptions()[msgName]:
description = self.fieldDescriptions()[msgName][fieldName]
elif fieldMsgDefault is not None and fieldMsgDefault in self.messageDescriptions():
# field description based on message description
description = self.messageDescriptions()[fieldMsgDefault]
return description
Barista/backend/caffe/proto_info.py 0000664 ڊ0011117 00000054366 13265130053 017565 0 ustar klemms o0cvpr from google.protobuf.descriptor import FieldDescriptor
import sys, inspect
from backend.caffe import proto_description
import re
import copy
"""This module dynamically provides meta information about the available caffe parameters.
If the compiled caffe version is changed, the provided meta information will change, too.
Use these information to show available options in the gui.
Example to get all layer types: types = CaffeMetaInformation().availableLayerTypes()
"""
moreLayerNameParameter = {
"Deconvolution": "ConvolutionParameter"
}
class Singleton(type):
"""This metaclass is used to provide the singleton pattern in a generic way.
See http://stackoverflow.com/a/6798042 for source and further explanation.
TODO outsource this class to use the same pattern in the complete project?
"""
_instances = {}
def __call__(cls, *args, **kwargs):
if cls not in cls._instances:
cls._instances[cls] = super(Singleton, cls).__call__(*args, **kwargs)
return cls._instances[cls]
class UnknownLayerTypeException(Exception):
def __init__(self, msg):
self._msg = msg
def __str__(self):
return self._msg
class CaffeMetaInformation:
"""This class is the main interface for any usage outside of this module.
It provides various meta information about the compiled caffe version and its available features.
Singleton is used to ensure that all constant data is generated only once.
"""
__metaclass__ = Singleton
def __init__(self):
self.updateCaffeMetaInformation()
def updateCaffeMetaInformation(self):
self.__initAvailableParameterGroupDescriptors()
self.__initAvailableLayerTypes()
self.__initAvailableSolverTypes()
def __initAvailableParameterGroupDescriptors(self):
"""Generate information about available parameter only once using this method.
See description of self.availableParameterGroupDescriptors().
"""
from backend.caffe.path_loader import PathLoader
proto = PathLoader().importProto()
current_module = sys.modules[proto.__name__]
res = {}
for (el,val) in inspect.getmembers(current_module, inspect.isclass):
res[el] = ParameterGroupDescriptor(val)
self._availableParameterGroupDescriptors = res
def __initAvailableLayerTypes(self):
"""Generate information about available layer types only once.
self.__initAvailableParameterGroupDescriptors() needs to be called before this method."""
from backend.caffe.path_loader import PathLoader
caffe = PathLoader().importCaffe()
layerNameMainParts = list(caffe.layer_type_list())
res = {}
paramsPerLayerType = {}
# calculate common parameters of all layer types
# by removing all which will be used for one specific layer type only
# also keep in mind which ones have been removed to readd them to specific layers
commonParams = self._availableParameterGroupDescriptors["LayerParameter"].parameter() #use .parameter() on purpose
layerSpecificParameters = set()
for nameMainPart in layerNameMainParts:
specificParamsName = [nameMainPart + "Parameter"]
if moreLayerNameParameter.has_key(nameMainPart):
specificParamsName.append( moreLayerNameParameter[nameMainPart])
paramsPerLayerType[nameMainPart] = {}
for key, value in commonParams.items():
if value.isParameterGroup() and value.parameterName() in specificParamsName:
paramsPerLayerType[nameMainPart][key] = value
layerSpecificParameters.add(key)
# special case: shared params for loss layers
key = "loss_param"
value = commonParams[key]
del commonParams[key]
for nameMainPart in layerNameMainParts:
if LayerType.getCategoryByName(nameMainPart) == LayerType.CATEGORY_LOSS:
paramsPerLayerType[nameMainPart][key] = value
# TODO is there a special case for the TransformationParameter?
# create each layer type after one another
for nameMainPart in layerNameMainParts:
# add common params to the specific ones
layerTypeParam = paramsPerLayerType[nameMainPart].keys()
paramsPerLayerType[nameMainPart].update(commonParams)
irrelevant = layerSpecificParameters.difference(layerTypeParam)
res[nameMainPart] = LayerType(nameMainPart, paramsPerLayerType[nameMainPart], layerTypeParam, irrelevant)
self._commonParams = commonParams
self._availableLayerTypes = res
def __initAvailableSolverTypes(self):
"""Generate information about available solver types only once.
self.__initAvailableParameterGroupDescriptors() needs to be called before this method."""
from backend.caffe.path_loader import PathLoader
proto = PathLoader().importProto()
# DO NOT REMOVE the following import statement, although your IDE might say it's unused. It's not!
from caffe._caffe import Solver as SolverBaseClassInfo
# Unfortunately, there isn't a preexisting method to get all solver (names).
# But, there is a base class which is subclassed by all existing solvers.
# Use those subclasses to gain the names of all solvers.
solverNamesFull = [cls.__name__ for cls in vars()['SolverBaseClassInfo'].__subclasses__()]
# All those names end with the suffix "Solver".
# Remove that suffix to be consistent with the names of available LayerTypes.
solverNameMainParts = []
for name in solverNamesFull:
solverNameMainParts.append(name.replace("Solver",""))
# Get all (common) params of a solver type
# TODO try to separate params which should be available only for a specific SolverType.
# Unfortunately, there doesn't seem to exist any way to automatically do that. There are only a few comments
# in the caffe.proto file which would allow hardcoding some special cases. But is that a good idea?
commonParams = self._availableParameterGroupDescriptors[
"SolverParameter"].parameter() # use .parameter() on purpose
# create each solver type after one another
res = {}
for nameMainPart in solverNameMainParts:
res[nameMainPart] = SolverType(nameMainPart, commonParams)
self._availableSolverTypes = res
def availableParameterGroupDescriptors(self):
"""Return all available Parameter (e.g. Layer-Parameter) for Caffe as an dictionary.
The key is the name of the parameter and the value is an instance of the helper class ParameterGroupDescriptor.
This is the most general way of getting any caffe meta information.
"""
return self._availableParameterGroupDescriptors
def availableLayerTypes(self):
"""Return a dictionary containing all layer types which are available in caffe.
The key is the name of the layer type (without the suffix 'Layer'. e.g. 'SoftMax' instead of 'SoftMaxLayer')
and the value is an object of the class LayerType.
"""
return self._availableLayerTypes
def getLayerType(self, typename):
""" Returns the layer type for typename or throws an UnknownLayerTypeException
if a type with typename is not known. """
if typename not in self._availableLayerTypes:
raise UnknownLayerTypeException("Network contains an unknown layer type '{}'.".format(typename))
return self._availableLayerTypes[typename]
def commonParameters(self):
""" Return parameters all layers share.
It is missing specific paremeters like "inner_product_param"
or "convolution_param".
"""
return self._commonParams
def availableSolverTypes(self):
"""Return a dictionary containing all solver types which are available in caffe.
The key is the name of the solver type (without the suffix 'Solver'. e.g. 'SGD' instead of 'SGDSolver')
and the value is an object of the class SolverType.
"""
return self._availableSolverTypes
def availableNetParameters(self):
"""Return a list containing all available parameters of a net.
Note that, this differs a little bit from the way you get the parameters of a solver or layer, as we don't
got different net types.
Also pay attention to the fact, that the returned list will of course contain the parameter "layer". Because
this parameter is only a general description of all layers at once, you will probably want to use
availableLayerTypes() instead.
"""
return self._availableParameterGroupDescriptors['NetParameter'].parameter()
class TopLevelEntityType:
"""This class represents entities of special importance for our project.
According to the caffe.proto file, those entities are normal ParameterGroupDescriptor (/Message) docks.
But the proto file doesn't add a hierarchy for the top level. This hierarchy will be added by inheriting from this
class.
The already complete list of top level entities is probably: LayerType, SolverType
"""
def __init__(self, name, parameters):
self._name = name
self._parameters = parameters
def name(self):
"""The name extended by the correct suffix (e.g. 'Layer') equals the name of its C++ class."""
return self._name
def parameters(self):
"""Return a dictionary containing all available parameters for this TopLevelEntityType.
Each parameter will be an instance of a subclass of Parameter.
"""
return self._parameters
class LayerType(TopLevelEntityType):
"""An object of this class represents one type of layer that is available in caffe.
The name of such an object is for instance "Softmax" or "InnerProduct".
The parameters of a LayerType include general parameters which are available for all LayerTypes as well as specific
ones which are only available for this particular LayerType.
"""
CATEGORY_NONE = 0
CATEGORY_LOSS = 1
CATEGORY_DATA = 2 # == input
CATEGORY_ACTIVATION = 3
# TODO (try to) implement further categories
def __init__(self, name, parameters, layerParamKeys=[], seemsNotRelevantParams=set()):
TopLevelEntityType.__init__(self, name, parameters)
self._category = self.getCategoryByName(self._name)
self._layerParamKeys = layerParamKeys
self._irrelevant = seemsNotRelevantParams
def category(self):
"""Return the category of this LayerType.
The category is described by one of the class constants CATEGORY_.
"""
return self._category
def layerParamKeys(self):
"""The paremter key for the specific parmeters of this Layertype
E.g. ["inner_product_param"] for the InnerProduct-Layer
"""
return self._layerParamKeys
def probablyIrrelevantParameter(self):
""" Return a set of parameter which seems not to belong to this layer """
return self._irrelevant
def isLossLayer(self):
return self._category == self.CATEGORY_LOSS
def isDataLayer(self):
return self._category == self.CATEGORY_DATA
def isActivationLayer(self):
return self._category == self.CATEGORY_ACTIVATION
def allowsInPlace(self):
inPlaceLayers = ['Dropout']
return self._name in inPlaceLayers or self.isActivationLayer()
@staticmethod
def getCategoryByName(layerTypeName):
"""Given its name, determine which category a LayerType does belong to."""
# !!! hard coded list of activation layers could cause problems when more layers are added in caffe
if layerTypeName in ["ReLU", "PReLU", "ELU", "Sigmoid", "TanH", "AbsVal", "Power",
"Exp", "Log", "BNLL", "Threshold", "Bias", "Scale"]:
return LayerType.CATEGORY_ACTIVATION
elif "Loss" in layerTypeName:
return LayerType.CATEGORY_LOSS
elif "Data" in layerTypeName:
return LayerType.CATEGORY_DATA
else:
return LayerType.CATEGORY_NONE
def __copy__(self):
"""See __deepcopy__."""
result = CaffeMetaInformation().availableLayerTypes()[self._name]
return result
def __deepcopy__(self, memo):
"""Override the deepcopy behavior used in copy.deepcopy() and pickle().
The protobuf object stored in self._field can't be pickeled, which will also result in an exception each time we
try to call copy.deepcopy() on any list/dict/etc. that contains an object of this class. We need to do this e.g.
in network_manager._refreshStateFromHistory(). This method solves the problem by just recreating an object with
the same information as the original one without copying anything else. That's actually everything we need.
"""
result = CaffeMetaInformation().availableLayerTypes()[self._name]
memo[id(self)] = result
return result
class SolverType(TopLevelEntityType):
"""Currently, the following SolverTypes do exist: 'SGD', 'Nesterov', 'AdaGrad', 'RMSProp', 'AdaDelta', 'Adam'"""
class ParameterGroupDescriptor:
"""A ParameterGroupDescriptor is a general description of a ParameterGroup without being used as a concrete parameter.
The idea is equivalent to a Message in the Google Protobuf definition.
"""
def __init__(self,protoclass, descriptor=None):
self._protoclass = protoclass
if descriptor is None and protoclass is not None:
self._desc = protoclass.DESCRIPTOR
else:
self._desc = descriptor
def protoclass(self):
""" Return the protoclass associated with this descriptor """
return self._protoclass
def name(self):
return self._desc.name
def parameter(self):
"""Return a dictionary containing all parameters of this group.
The parameter name is used as the key while each value is an instance of the class Parameter. This instance can
be used too gain further information.
"""
res = dict()
for x in self._desc.fields:
if x.type == FieldDescriptor.TYPE_MESSAGE:
parameter = ParameterGroup(x)
elif x.type == FieldDescriptor.TYPE_ENUM:
parameter = ParameterEnum(x)
else:
parameter = ParameterPrimitive(x)
res[x.name] = parameter
return res
def __eq__(self, other):
return self._desc.full_name == other._desc.full_name
class Parameter:
"""This class represents a single parameter which can be specified in the .prototxt file.
There are various use cases for a parameter of this type.
It might be usable only for the net definition, the solver definition, or both.
A parameter also does not necessarily need to be a top-level parameter, meaning it might only be used
as the value of another parameter, too.
A Parameter is also a single property of a ParameterGroupDescriptor and therefore sometimes referred to as a
'property', too.
An object of this (abstract) class will always be an instance of one of the following subclasses:
ParameterPrimitive, ParameterEnum or ParameterGroup.
TODO this base class should not be allowed to instantiate and should therefore be an abstract class,
but that doesn't make sense in python, does it?
"""
string_dict = {
FieldDescriptor.TYPE_BOOL: "bool",
FieldDescriptor.TYPE_BYTES: "bytes",
FieldDescriptor.TYPE_DOUBLE: "double",
FieldDescriptor.TYPE_ENUM: "enum",
FieldDescriptor.TYPE_FIXED32: "fixed32",
FieldDescriptor.TYPE_FIXED64: "fixed64",
FieldDescriptor.TYPE_FLOAT: "float",
# FieldDescriptor.TYPE_GROUP: "group",
FieldDescriptor.TYPE_INT32: "int32",
FieldDescriptor.TYPE_INT64: "int64",
FieldDescriptor.TYPE_MESSAGE: "message",
FieldDescriptor.TYPE_SFIXED32: "sfixed32",
FieldDescriptor.TYPE_SFIXED64: "sfixed64",
FieldDescriptor.TYPE_SINT32: "sint32",
FieldDescriptor.TYPE_SINT64: "sint64",
FieldDescriptor.TYPE_STRING: "string",
FieldDescriptor.TYPE_UINT32: "uint32",
FieldDescriptor.TYPE_UINT64: "uint64",
}
def __init__(self,field):
self._field = field
self._description = proto_description.CaffeProtoParser().description(field.containing_type.name, field.name)
self._initIsDeprecated()
def _initIsDeprecated(self):
"""Call this whenever a new description has been set."""
self._isDeprecated = re.search("deprecated", self._description, re.IGNORECASE) is not None
def isDeprecated(self):
return self._isDeprecated
def description(self):
"""Get a string describing this parameter in a human-readable way."""
return self._description
def fieldName(self):
"""Return the name of this property (with underscores, not camelcase).
This is not the same as the name of the parameter's type. Therefore this method is not called 'name()'.
Pay additional attention to possible confusion with the name of the ParameterGroupDescriptor, if this element
is a ParameterGroup.
"""
return self._field.name
def isOptional(self):
"""Return true iff the parameter is optional, this includes if it's repeated.
(Repeated mean 0-n)
"""
return self._field.label == FieldDescriptor.LABEL_OPTIONAL or self.isRepeated()
def isRequired(self):
""" Return the opposite of isOptional"""
return not self.isOptional()
# return self._field.label == FieldDescriptor.LABEL_REQUIRED
def isRepeated(self):
""" Return true iff the parameter is repeated. Repeated values may contain 0 entries"""
return self._field.label == FieldDescriptor.LABEL_REPEATED
def isParameterGroup(self):
"""Return true iff this parameter is a combination of other parameter."""
return self._field.type == FieldDescriptor.TYPE_MESSAGE
def defaultValue(self):
# We make a copy, so no one can override the default value
return copy.deepcopy(self._field.default_value)
def __eq__(self, other):
return self._field.full_name == other._field.full_name and self.fieldName() == other.fieldName()
def type(self):
"""The type is descript by one of the constants FieldDescriptor.TYPE_."""
return self._field.type
def typeString(self):
"""Return a human-readable string describing the type (e.g. "bool", "int32", ..). """
return Parameter.string_dict[self._field.type]
def isBool(self):
return self.type() == FieldDescriptor.TYPE_BOOL
def isString(self):
return self.type() == FieldDescriptor.TYPE_STRING
def isInt(self):
"""Combine all possible tests of integer (does not differ: uint vs int vs sint)."""
return self.type() == FieldDescriptor.TYPE_INT32 or self.type() == FieldDescriptor.TYPE_INT64 or \
self.type() == FieldDescriptor.TYPE_SINT32 or self.type() == FieldDescriptor.TYPE_SINT64 or \
self.type() == FieldDescriptor.TYPE_UINT32 or self.type() == FieldDescriptor.TYPE_UINT64 or \
self.type() == FieldDescriptor.TYPE_FIXED32 or self.type() == FieldDescriptor.TYPE_FIXED64 or \
self.type() == FieldDescriptor.TYPE_SFIXED32 or self.type() == FieldDescriptor.TYPE_SFIXED64
def isSInt(self):
return self.type() == FieldDescriptor.TYPE_SINT32 or self.type() == FieldDescriptor.TYPE_SINT64 or \
self.type() == FieldDescriptor.TYPE_SFIXED32 or self.type() == FieldDescriptor.TYPE_SFIXED64
def isUInt(self):
return self.type() == FieldDescriptor.TYPE_UINT32 or self.type() == FieldDescriptor.TYPE_UINT64 or \
self.type() == FieldDescriptor.TYPE_FIXED32 or self.type() == FieldDescriptor.TYPE_FIXED64
def isBytes(self):
return self.type() == FieldDescriptor.TYPE_BYTES
def isFloat(self):
return self.type() == FieldDescriptor.TYPE_FLOAT
def isDouble(self):
return self.type() == FieldDescriptor.TYPE_DOUBLE
def isEnum(self):
return self.type() == FieldDescriptor.TYPE_ENUM
class ParameterPrimitive(Parameter):
"""An Object of this class represents a primitive data type like int, bool, string etc."""
class ParameterEnum(Parameter):
"""An Object of this class represents an enumeration.
TODO throw exception in constructor if self._field.type != FieldDescriptor.TYPE_ENUM
"""
def availableValues(self):
"""Return all available values possible for this enum."""
return [x.name for x in self._field.enum_type.values]
def defaultValue(self):
return self.availableValues()[self._field.default_value]
def resetCaffeProtoModulesvar():
global _caffeprotomodulesvar
_caffeprotomodulesvar = None
def _caffeProtobufModules():
""" Returns all available Classes of caffe_pb2 in a dictionary """
from backend.caffe.path_loader import PathLoader
proto = PathLoader().importProto()
global _caffeprotomodulesvar
if _caffeprotomodulesvar is None:
current_module = sys.modules[proto.__name__]
_caffeprotomodulesvar = dict(inspect.getmembers(current_module, inspect.isclass))
return _caffeprotomodulesvar
class ParameterGroup(ParameterGroupDescriptor, Parameter):
"""An Object of this class is a ParameterGroupDescriptor which is used as a Parameter of another ParameterGroupDescriptor.
TODO throw exception in constructor if self._field.type != FieldDescriptor.TYPE_MESSAGE:
"""
def __init__(self, field):
Parameter.__init__(self, field)
ParameterGroupDescriptor.__init__(self, None, descriptor=field.message_type)
self._protoclass = _caffeProtobufModules()[self.parameterName()]
# refresh description as this type of parameter got additional default values
self._description = proto_description.CaffeProtoParser().description(field.containing_type.name, field.name,
field.message_type.name)
self._initIsDeprecated()
def parameterName(self):
"""Get the ParameterGroupDescriptor name.
Just an alias for the name() method to prevent confusion caused by the polymorphism.
"""
return self.name()
Barista/backend/caffe/saver.py 0000664 ڊ0011117 00000007036 13265130053 016517 0 ustar klemms o0cvpr from google.protobuf import text_format
import backend.caffe.proto_info as info
def saveSolver(solverdict):
solver= _import_solver(solverdict)
solverproto = text_format.MessageToString(solver)
return solverproto
def _import_solver(solverdict):
from backend.caffe.path_loader import PathLoader
proto = PathLoader().importProto()
solver = proto.SolverParameter()
for entry in solverdict:
# special case: inline-net definition in solver
# (required to handle layers and layerOrder the same way it is handled for a standalone net definition)
if entry == "net_param":
net = _import_dictionary(solverdict["net_param"])
solver.net_param.MergeFrom(net)
else:
_insert(entry, solverdict[entry], solver)
return solver
def saveNet(netdict):
""" Save the dictionary into a prototxt string.
The dictionary has the following form
{
"name": "Somenetwork",
"input_dim": [1,2,1,1],
"state": {
"phase": "TRAIN"
},
...
"layers":
{
"somerandomid1": {
"type": LayerType Instance of Pooling-Layer,
"parameters": {
"pooling_param": [
"kernel_size": 23,
"engine": "DEFAULT"
]
....
"input_param": [
{"shape": {"dim": [...], .... },
{"shape": {"dim": [...], .... },
]
}
},
"somerandomid2": {"type": ..., "parameters": ....}
},
"layerOrder": ["somerandomid1", "somerandomid2", ....]
}
"""
net = _import_dictionary(netdict)
netproto = text_format.MessageToString(net)
return netproto
def _import_dictionary(netdict):
"""fill the ProtoTxt-Net with data from the dictionary"""
from backend.caffe.path_loader import PathLoader
proto = PathLoader().importProto()
net = proto.NetParameter()
for entry in netdict:
if entry == "layerOrder":
continue
if entry == "layers":
_extract_layer(netdict["layers"],netdict["layerOrder"],net)
continue
_insert(entry,netdict[entry],net)
return net
def _extract_layer(layers, layerorder,net):
for order in layerorder:
index = len(net.layer)
net.layer.add()
currlayer = layers[order]
for param in currlayer["parameters"]:
_insert(param,currlayer["parameters"][param],net.layer[index])
return
def _insert(key, value, insert):
descr = info.ParameterGroupDescriptor(insert)
param = descr.parameter()[key]
if param.isParameterGroup():
if param.isRepeated():
for l in range(0, len(value)):
index = len(getattr(insert,key))
getattr(insert,key).add()
for v in value[l]:
_insert(v,value[l][v],getattr(insert,key)[index])
else:
for v in value:
_insert(v,value[v],getattr(insert,key))
return
if param.isRepeated():
for v in value:
w = v
if param.isEnum():
w = param.availableValues().index(v)
getattr(insert,key).append(w)
return
if param.isEnum():
value = param.availableValues().index(value)
setattr(insert, key, value)
Barista/backend/input_db/__init__.py 0000664 ڊ0011117 00000000000 13265130053 017656 0 ustar klemms o0cvpr Barista/backend/input_db/hdf5_input.py 0000664 ڊ0011117 00000005121 13265130053 020175 0 ustar klemms o0cvpr import os.path
import h5py as h5
import numpy as np
from backend.barista.utils.logger import Log
class Hdf5Input:
def __init__(self, pathOfHdf5Txt=False):
self._db = None
self._path = None
self._pathOfHdf5Txt = pathOfHdf5Txt # HDF5TXT files can contain commentary lines that are no paths
self._logid = Log.getCallerId('HDF5 Input')
def __del__(self):
self.close()
def setPath(self, path):
'''set the path of the database.'''
self._path = path
def getPath(self):
'''get the path of this HDF5Input object'''
return self._path
def open(self):
'''open the database from the set path.'''
if self._db:
self.close()
if self._path:
if os.path.exists(self._path):
try:
self._db = h5.File(self._path, 'r')
except:
Log.error("File not valid HDF5: " + self._path, self._logid)
self._db = None
elif not self._pathOfHdf5Txt:
Log.error("File does not exist: " + self._path, self._logid)
def close(self):
if self._db:
self._db.close()
self._db = None
def getDataCount(self):
if self._db:
d = {}
for i in range(0, len(self._db.keys())):
d[self._db.keys()[i]] = len(self._db.values()[i])
return d
def getDimensions(self):
if self._db:
d = {}
for i in range(0, len(self._db.keys())):
ar = np.array(self._db.values()[i][0])
d[self._db.keys()[i]] = ar.shape
return d
def verifyConsistency(self):
if self._db:
for i in range(0, len(self._db.keys())):
ar = np.array(self._db.values()[i][0])
for j in range(0, len(self._db.values()[i])):
ar2 = np.array(self._db.values()[i][j])
if not self._dimCompare(ar.shape, ar2.shape):
return False
return True
def isOpen(self):
if self._db:
return True
return False
def _dimCompare(self, d1, d2):
if not d1 and not d2: # both empty
return True
if d2 and not d1: # d1 empty
return False
if d1 and not d2: # d2 empty
return False
if len(d1) != len(d2): # different dim count
return False
for i in range(0, len(d1)): # dims not matching
if d1[i] != d2[i]:
return False
return True
Barista/backend/input_db/hdf5_txt_input.py 0000664 ڊ0011117 00000006153 13265130053 021102 0 ustar klemms o0cvpr from os import path
from backend.barista.utils.logger import Log
from backend.input_db.hdf5_input import *
class Hdf5TxtInput:
def __init__(self):
self._db = []
self._path = None
self._logid = Log.getCallerId("HDF5TXT Input")
self._projectPath = None
def __del__(self):
self.close()
def setPath(self, path):
self._path = os.path.normpath(path)
def open(self):
if self._db:
self.close()
if self._path is not None:
if os.path.exists(self._path):
lines = [line.rstrip('\n') for line in open(self._path)]
hdf5Count = 0
for line in lines:
if line is not "":
if line[:1] == '.':
line = self._makepath(line)
i = len(self._db)
self._db.append(Hdf5Input(pathOfHdf5Txt=True))
self._db[i].setPath(line)
self._db[i].open()
if self._db[i].isOpen():
hdf5Count += 1
if hdf5Count == 0:
self._db = None
Log.error("File contained no valid paths to HDF5 files: {}".format(self._path), self._logid)
def close(self):
if self._db:
for db in self._db:
db.close()
self._db = []
def getDataCount(self):
if self._db:
sum = {}
for db in self._db:
sum = self._sumDataCount(sum, db.getDataCount())
return sum
def getDimensions(self):
if self._db:
d = {}
for db in self._db:
d = self._combineDim(d, db.getDimensions())
return d
def verifyConsistency(self):
if self._db:
d = self._db[0].getDimensions()
for db in self._db:
if not db.verifyConsistency():
return False
if not self._compareDim(d, db.getDimensions()):
return False
return True
def isOpen(self):
if self._db:
return True
return False
def setProjectPath(self, path):
self._projectPath = path
def _sumDataCount(self, d1, d2):
dataCountDict = {}
if d1 or d2:
dataCountDict = {k: d1.get(k, 0) + d2.get(k, 0) for k in set(d1) | set(d2)}
return dataCountDict
def _combineDim(self, d1, d2):
return {k: max(d1.get(k, 0), d2.get(k, 0)) for k in set(d1) | set(d2)}
def _compareDim(self, d1, d2):
l = [d1.get(k) == d2.get(k) for k in set(d1) | set(d2)]
if False in l:
return False
return True
"""
Converts a path that is considered to be relative to the project path into an absolute path.
@author: ..., j_stru18
@param relpath a path that should be relative to the project path
"""
def _makepath(self, relpath):
if self._projectPath:
p = path.abspath(path.join(self._projectPath, relpath))
return p
return relpath
Barista/backend/input_db/leveldb_input.py 0000664 ڊ0011117 00000006206 13265130053 020771 0 ustar klemms o0cvpr import leveldb # pip install leveldb
import os.path
from backend.barista.utils.logger import Log
class LeveldbInput:
def __init__(self):
self._db = None
self._path = None
self.logid = Log.getCallerId("LEVELDB Input")
def __del__(self):
self.close()
def setPath(self, path):
'''set the path of the database.'''
self._path = path
def open(self):
'''open the database from the set path.'''
if self._db:
self.close()
if self._path:
if os.path.exists(self._path + "CURRENT") or os.path.exists(self._path + "/CURRENT"):
self._db = leveldb.LevelDB(self._path)
else:
Log.error("Dir is not valid LEVELDB: " + self._path, self.logid)
def close(self):
'''close and clean up the database.'''
self._db = None
def getDataCount(self):
'''get the number of entries in the database'''
if self._db:
iter = self._getIter()
if iter:
number = 0
for elem in iter:
number = number + 1
return {"data": number}
def getDimensions(self):
'''get the dimensions of the first entry'''
datum = self._getFirstDatum()
if datum:
dim = (datum.channels, datum.height, datum.width)
return {"label": (), "data": dim} # TODO dim of label
def verifyConsistency(self):
'''check if all entries have the same channel number and size.
This may take a lot of time since every entry has to be checked.'''
from backend.caffe.path_loader import PathLoader
caffe = PathLoader().importCaffe()
if self.getDataCount() < 2:
return True
first = self._getFirstDatum()
if first:
channels = first.channels
width = first.width
height = first.height
iter = self._getIter()
if iter:
for key, value in iter:
raw_datum = value
datum = caffe.proto.caffe_pb2.Datum()
datum.ParseFromString(raw_datum)
if channels is not datum.channels \
or width is not datum.width \
or height is not datum.height:
return False
return True
def isOpen(self):
'''check if db is open.'''
if self._db:
return True
return False
def _getIter(self):
if self._db:
return self._db.RangeIter()
def _getFirstDatum(self):
from backend.caffe.path_loader import PathLoader
caffe = PathLoader().importCaffe()
iter = self._getIter()
if iter:
for key, value in iter:
raw_datum = value
datum = caffe.proto.caffe_pb2.Datum()
try:
datum.ParseFromString(raw_datum)
except:
Log.error("LEVELDB does not contain valid data: " + self._path, self.logid)
return None
return datum
Barista/backend/input_db/lmdb_input.py 0000664 ڊ0011117 00000006505 13265130053 020274 0 ustar klemms o0cvpr import os.path
import lmdb # pip install lmdb
from backend.barista.utils.logger import Log
class LmdbInput:
def __init__(self):
self._env = None
self._path = None
self._cursor = None
self._txn = None
self.logid = Log.getCallerId("LMDB Input")
def __del__(self):
self.close()
def setPath(self, path):
'''set the path of the database.'''
self._path = path
def open(self):
'''open the database from the set path.'''
if self._env:
self.close()
if self._path:
if os.path.exists(self._path + "data.mdb") or os.path.exists(self._path + "/data.mdb"):
self._env = lmdb.open(self._path, max_dbs=2)
else:
Log.error("Dir is not valid LMDB: " + self._path, self.logid)
def close(self):
'''close and clean up the database.'''
if self._env:
self._env.close()
self._env = None
self._txn = None
self._cursor = None
def getDataCount(self):
'''get the number of entries in the database'''
if self._env:
return {"data": self._env.stat()['entries']}
def getDimensions(self):
'''get the dimensions of the first entry'''
if self._env:
datum = self._getFirstDatum()
if datum:
dim = (datum.channels, datum.height, datum.width)
return {"label": (), "data": dim} # TODO dim of label
def verifyConsistency(self):
'''check if all entries have the same channel number and size.
This may take a lot of time since every entrie has to be checked.'''
if self._env:
if self.getDataCount() < 2:
return True
first = self._getFirstDatum()
if first:
channels = first.channels
width = first.width
height = first.height
while self._cursor.next():
if channels is not self._getCurrentDatum().channels \
or width is not self._getCurrentDatum().width \
or height is not self._getCurrentDatum().height:
return False
return True
def isOpen(self):
'''check if db is open:'''
if self._env:
return True
return False
def _openTransaction(self):
if self._env:
if self._txn is None:
self._txn = self._env.begin()
def _openCursor(self):
if self._env:
if self._txn:
if self._cursor is None:
self._cursor = self._txn.cursor()
def _getFirstDatum(self):
self._openTransaction()
self._openCursor()
if self._cursor:
self._cursor.first()
return self._getCurrentDatum()
def _getCurrentDatum(self):
from backend.caffe.path_loader import PathLoader
caffe = PathLoader().importCaffe()
if self._cursor:
raw_datum = self._cursor.value()
datum = caffe.proto.caffe_pb2.Datum()
try:
datum.ParseFromString(raw_datum)
except:
Log.error("LMDB does not contain valid data: " + self._path, self.logid)
return None
return datum
Barista/backend/networking/__init__.py 0000664 ڊ0011117 00000000000 13265130053 020241 0 ustar klemms o0cvpr Barista/backend/networking/barista_server.py 0000664 ڊ0011117 00000015325 13265130053 021535 0 ustar klemms o0cvpr import json
import os
import socket
import sys
import logging
from PyQt5.QtNetwork import QTcpServer, QHostAddress
from backend.caffe.check_hardware import checkHardware
from backend.barista.session.session_utils import State
from backend.networking.protocol import Protocol
from backend.networking.server_session_manager import ServerSessionManager
from backend.networking.server_transaction import ServerTransaction
from backend.barista import caffe_versions
class BaristaServer():
MIN_PORT = 0
MAX_PORT = 65535
DEFAULT_PORT = 4200
def __init__(self, application, ip, port, sessionPath):
self.application = application
self.server = None
self.port = int(port)
self.ip = QHostAddress.Any
if ip is not None:
self.ip = QHostAddress(ip)
self.sessionPath = sessionPath
self.configpath = os.path.join(self.sessionPath, "barista.conf") ####
self.hardware = []
self.trainOnHW = 0
self.transactionList = []
self._loadConfig()
self.checkHardware()
state = self._checkConfig()
self.start()
self.sessionManager = ServerSessionManager(self, self.sessionPath)
def start(self):
self.server = QTcpServer()
self.server.listen(self.ip, self.port)
self.server.newConnection.connect(self._newConnection)
if self.server.isListening():
hostIP = str(self.server.serverAddress().toString())
if hostIP == '0.0.0.0':
hostIP = ''
sys.stdout.write("Hostname: " + socket.gethostname()
+ "\tIP: " + hostIP + ":" + str(self.server.serverPort()) + "\n")
else:
sys.stderr.write("Something went wrong. Server is not listening.\n")
logging.error("Something went wrong. Server is not listening.")
exit(1)
def getBaristaStatus(self, pid=""):
hardware = [k["name"] for k in self.hardware]
statusdict = {"trainOnHW": self.trainOnHW, "hardware": hardware}
statusdict["connections"] = len(self.transactionList)
statusdict["config"] = self._checkConfig(False)
statusdict["training"] = len(self.sessionManager.findSessionIDsWithState(State.RUNNING)) is 0
statusdict["sessioncount"] = len(self.sessionManager.sessions)
statusdict["sessionpath"] = self.sessionPath
if pid is not "":
pidses = {}
sessions = set(self.sessionManager.findSessionIDsByProjectId(pid))
pidses["pid"] = pid
pidses["count"] = len(sessions)
pidses["running"] = len(set(self.sessionManager.findSessionIDsWithState(State.RUNNING)) & sessions)
pidses["running"] += len(set(self.sessionManager.findSessionIDsWithState(State.PAUSED)) & sessions)
pidses["waiting"] = len(set(self.sessionManager.findSessionIDsWithState(State.WAITING)) & sessions)
pidses["finished"] = len(set(self.sessionManager.findSessionIDsWithState(State.FINISHED)) & sessions)
statusdict["projectsessions"] = pidses
return statusdict
def _newConnection(self):
transaction = ServerTransaction(self)
self.transactionList.append(transaction)
transaction.socketClosed.connect(lambda item=transaction: self._deleteConnection(item))
transaction.acceptClient(self.server.nextPendingConnection())
def _deleteConnection(self, transaction):
index = self.transactionList.index(transaction)
del self.transactionList[index]
def _saveConfig(self):
#Save train on hardware
tosave = dict()
tosave["trainOnHW"] = self.trainOnHW
with open(self.configpath, "w") as file:
json.dump(tosave, file, sort_keys=True, indent=4)
def _loadConfig(self):
####
if not os.path.exists(self.configpath):
# TODO Default
pass
else:
with open(self.configpath, "r") as file:
res = json.load(file)
if "trainOnHW" in res:
self.trainOnHW = res["trainOnHW"]
else:
# TODO default
pass
def _checkConfig(self, verbose=True):
state = True
caffe_versions.loadVersions(self.sessionPath)
if caffe_versions.versionCount() == 0:
if verbose:
sys.stderr.write("Warning: There is no Caffeversion set. Use the Versionmanager inside Barista to set the Path.\n")
logging.warning("There is no Caffeversion set. Use the Versionmanager inside Barista to set the Path.")
if self.trainOnHW >= len(self.hardware):
self.trainOnHW = 0
if verbose:
sys.stderr.write("Warning: Currently selected Hardware could not be matched against detected Hardware."
" Set to CPU mode!\n")
logging.warning("Currently selected Hardware could not be matched against detected Hardware."
" Set to CPU mode!")
return state
def checkHardware(self, verbose=True, transaction=None):
caffe_versions.loadVersions(self.sessionPath)
if caffe_versions.versionCount() == 0:
if verbose:
sys.stderr.write("Warning: Can't check hardware without Caffeversions\n")
logging.warning("Can't check hardware without Caffeversions")
if transaction:
msg = {"key": Protocol.SCANHARDWARE, "status": False, "error": "Can't check hardware without Caffeversions"}
transaction.send(msg)
return
try:
binary = caffe_versions.getDefaultVersion().getBinarypath()
self.hardware = checkHardware(binary, not verbose, transaction)
if verbose:
sys.stdout.write("Finished scanning Hardware. " + str(len(self.hardware)) + " devices found.\n")
logging.info("Finished scanning Hardware. %s devices found.", str(len(self.hardware)))
if transaction:
msg = {"key": Protocol.SCANHARDWARE, "status": True, "finished": True, "hardware": self.hardware,
"current": self.trainOnHW}
transaction.send(msg)
except:
if verbose:
sys.stderr.write("Error: Failed to check hardware!\n")
logging.error("Failed to check hardware!")
if transaction:
msg = {"key": Protocol.SCANHARDWARE, "status": False, "error": "Failed to check hardware!"}
transaction.send(msg)
def setHardware(self, hid):
if hid >= len(self.hardware):
return False
self.trainOnHW = hid
self._saveConfig()
return True
Barista/backend/networking/client_transaction.py 0000664 ڊ0011117 00000001213 13265130053 022374 0 ustar klemms o0cvpr from PyQt5.QtCore import pyqtSignal
from backend.networking.transaction import Transaction
from backend.networking.protocol import Protocol
class ClientTransaction(Transaction):
showDirSig = pyqtSignal(list, bool)
def __init__(self):
Transaction.__init__(self)
# self.options = {Protocol.ECHO: self.msgEcho}
def processMessage(self):
"""process the received message and call further processing"""
pass
# msg = self.messageOutput[0]
# if msg["key"] in self.options:
# self.options[msg["key"]]()
# def msgEcho(self):
# msg = self.asyncRead()
# print(msg)
Barista/backend/networking/net_util.py 0000664 ڊ0011117 00000001373 13265130053 020343 0 ustar klemms o0cvpr from backend.barista.utils.logger import *
from backend.networking.client_transaction import ClientTransaction
def sendMsgToHost(host, port, msg):
ct = ClientTransaction()
ct.connect(host, port)
ct.waitForConnection()
if ct.isConnected():
ct.send(msg)
ret = ct.asyncRead()
ct.close()
if ret:
return ret
else:
Log.error("No answer from " + host + ":" + str(port), Log.getCallerId("Network Connection"))
else:
Log.error("Failed to connect to " + host + ":" + str(port), Log.getCallerId("Network Connection"))
def buildTransaction(host, port):
ct = ClientTransaction()
ct.connect(host, port)
ct.waitForConnection()
if ct.isConnected():
return ct
Barista/backend/networking/protocol.py 0000664 ڊ0011117 00000003047 13265130053 020361 0 ustar klemms o0cvpr
class Protocol:
""" Pseudo Enumeration for the communication protocol
"""
# GENERAL
ECHO = 0
RESTART = 1
# GET
GETDIR = 10
GETSTATUS = 11
GETDBSTATUS = 12
GETFILEHASH = 13
GETDIRHASH = 14
# INPUTDB
MAKEPATHRELATIVE = 20
MAKEHDF5TXT = 22
ADDHDF5 = 23
GETFILECONTENT = 24
WRITEFILECONTENT = 25
ADDCAFFEVERSION = 26
GETHASH = 27
# HARDWARE
GETHARDWARE = 30
SCANHARDWARE = 31
SETHARDWARE = 32
GETPLATFORM = 33
# SESSION
SESSION = 40
CREATESESSION = 41
CONNECTTOSESSION = 42
GETSESSIONS = 43
DELETESESSION = 44
CLONESESSION = 45
DISCONNECTSESSION = 46
# CAFFE-VERSIONS
GETCAFFEVERSIONS = 50
ADDCAFFEVERSIONS = 51
SETCURRENTCAFFEVERSION = 52
REMOVECAFFEVERSION = 53
GETDEFAULTCAFFEVERSION = 54
GETCAFFERESTART = 55
class SessionProtocol:
""" Pseudo Enumeration for the communication protocol for subkeys in remote sessions
"""
GETITERATION = 0
GETMAXITERATION = 1
GETSNAPSHOTS = 2
GETSTATE = 3
SETMAXITERATION = 4
GETPRETRAINED = 5
SAVE = 10
CHECKFILES = 11
CHECKTRAINING = 12
GETSTATEDICT = 20
SETSTATEDICT = 21
PRINTLOG = 30
UPDATEPARSER = 31
UPDATESTATE = 32
UPDATEKEYS = 33
ADDSNAPSHOT = 34
PARSEHANDLE = 35
FETCHPARSERDATA = 36
START = 40
PAUSE = 41
PROCEED = 42
TAKESNAPSHOT = 43
LOADINTERNALNET = 50
LOADDEPLOYEDNET = 51
LOADNETPARAMETER = 52
LOADCAFFEMODEL = 53
RESET = 98
DELETE = 99
Barista/backend/networking/server_session_manager.py 0000664 ڊ0011117 00000037636 13265130053 023276 0 ustar klemms o0cvpr import os
import re
import shutil
import sys
import json
from datetime import datetime
import logging
from backend.barista.session.server_session import ServerSession
from backend.barista.session.session_pool import SessionPool
from backend.barista.session.session_utils import Paths, State
from backend.caffe import loader
from backend.caffe.proto_info import resetCaffeProtoModulesvar
from PyQt5.QtCore import pyqtSignal
from PyQt5.Qt import QObject
from backend.caffe.proto_info import UnknownLayerTypeException
import backend.caffe.proto_info as info
class ServerSessionManager(QObject):
poolEmptyJob = pyqtSignal(str)
jsonKeys = ("Iteration", "MaxIter", "NetworkState", "ProjectID", "SID", "SessionState", "UID")
def __init__(self, parent, sessionPath):
super(ServerSessionManager, self).__init__()
resetCaffeProtoModulesvar()
self.parent = parent
self.sessionPath = sessionPath
# TODO CaffeMetaInfo (use caffe/protopath)
self.sessions = []
self.loadSessions()
def connectToSession(self, transaction):
"""connect to an existing session with UID"""
msg = transaction.asyncRead()
msg["error"] = []
msg["status"] = False
if "uid" in msg:
session = self.findSessionBySessionUid(msg["uid"])
if session:
if session.connect(transaction):
msg["status"] = True
logging.info("Session connected.")
else:
msg["error"].append("ConnectToSession: Failed to connect. Session is already connected!")
logging.error("Failed to connect. Session is already connected")
else:
msg["error"].append("ConnectToSession: No session with UID '" + msg["uid"] + "' found.")
logging.error("No session with UID '%s' found.", msg["uid"])
else:
msg["error"].append("ConnectToSession: No UID provided.")
logging.error("No UID provided.")
transaction.send(msg)
def createNewSession(self, transaction):
"""create a new session with SID and PID"""
msg = transaction.asyncRead()
if "pid" in msg and "sid" in msg and "layers" in msg:
layers = msg["layers"]
try:
for layer in layers:
typename = info.CaffeMetaInformation().getLayerType(layer)
except UnknownLayerTypeException as e:
msg["status"] = False
msg["error"] = [e._msg]
logging.error("Could not create session. Unknown layer")
transaction.send(msg)
return
dirp = self._createDirName(msg["sid"])
uid = self._createSession(msg["pid"], dirp)
self.findSessionBySessionUid(uid).setInitialSid(msg["sid"])
msg["status"] = True
msg["uid"] = uid
logging.info("Session created with UID '%s'", uid)
else:
msg["status"] = False
msg["error"] = ["CreateNewSession: No PID or SID provided."]
logging.error("Could not create session. No PID or SID provided.")
transaction.send(msg)
def cloneSession(self, transaction):
"""
Creates a new session with the same net/solver etc. and a additional .caffemodel file with pretrained weights
"""
msg = transaction.asyncRead()
if "pid" in msg and "sid" in msg:
# create new session
dirp = self._createDirName(msg["sid"])
uid = self._createSession(msg["pid"], dirp) # note: this session is empty for now, net/solver will be loaded later
newSession = self.findSessionBySessionUid(uid)
newSession.setInitialSid(msg["sid"])
msg["uid"] = uid
if "old_uid" in msg and "old_solverstate" in msg:
# find the old session (where the caffemodel is extracted from) with help of its uid
oldSession = self.findSessionBySessionUid(msg["old_uid"])
# given solverstate, is used to locate and identify the caffemodel
oldSolverstate = msg["old_solverstate"]
del msg["old_uid"]
del msg["old_solverstate"]
if type(oldSolverstate) is not str and type(oldSolverstate) is not unicode:
# if no valid solverstate is given, take the last model created for this session
oldCaffemodel = oldSession.getLastModel()
else:
snapshotDir = os.path.join(oldSession.getSnapshotDirectory(), oldSolverstate)
oldCaffemodel = loader.getCaffemodelFromSolverstate(snapshotDir)
if oldSession and oldCaffemodel:
# locate caffemodel
oldSnapshotDir = oldSession.getSnapshotDirectory()
oldModelPath = os.path.join(oldSnapshotDir, oldCaffemodel)
else:
logging.error('Failed to create session!')
msg["error"] = ["No valid Session to clone from"]
msg["status"] = False
transaction.send(msg)
return
# create directories for the new session
newSnapshotDir = newSession.getSnapshotDirectory()
self.__ensureDirectory(newSession.getDirectory())
self.__ensureDirectory(newSnapshotDir)
if os.path.isdir(newSnapshotDir):
newCaffemodel = 'pretrained.caffemodel'
newModelPath = os.path.join(newSnapshotDir, newCaffemodel)
try:
# copy the old caffemodel to the new location
shutil.copy2(oldModelPath, newModelPath)
# initialize new session
newSession.setPretrainedWeights(newCaffemodel)
newSession.iteration = 0
newSession.max_iter = oldSession.max_iter
except Exception as e:
logging.error('Failed to copy caffemodel to new session: ' + str(e))
# copy the old state-dict into the new session
newSession.setStateDict(oldSession.state_dictionary)
newSession.setState(State.WAITING)
newSession.save(includeProtoTxt=True)
msg["status"] = True
logging.info("Session cloned with UID '%s'", uid)
else:
# cloning failed; clean up by deleting the new session
newSession.delete()
logging.error('Snapshot directory ' + newSnapshotDir + ' does not exist!')
msg["error"] = ["Snapshot directory does not exist!"]
msg["status"] = False
else:
msg["status"] = False
msg["error"] = ["CreateNewSession: No PID or SID provided."]
logging.error("Could not create session. No PID or SID provided.")
transaction.send(msg)
def __ensureDirectory(self, directory):
""" Creates a directory if it does not exist.
"""
if directory == '':
return
if not os.path.exists(directory):
try:
os.makedirs(directory)
logging.info('Created directory: '+directory)
except Exception as e:
logging.error('Failed to create directory '+directory+')'+str(e))
def deleteSession(self, transaction):
"""delete an existing session with UID"""
msg = transaction.asyncRead()
msg["error"] = []
msg["status"] = False
if "uid" in msg:
session = self.findSessionBySessionUid(msg["uid"])
if session:
for sess in self.sessions:
if sess["uid"] == msg["uid"]:
self.sessions.remove(sess)
session.delete()
del session
msg["status"] = True
logging.info("Session deleted.")
else:
msg["error"].append("DeleteSession: No session with UID '" + msg["uid"] + "' found.")
logging.error("No session with UID '%s' found.", msg["uid"])
else:
msg["error"].append("DeleteSession: No UID provided.")
logging.error("No UID provided.")
transaction.send(msg)
def disconnectSession(self, transaction):
msg = transaction.asyncRead()
if "uid" in msg:
session = self.findSessionBySessionUid(msg["uid"])
if session:
session.disconnect()
def getSessions(self, transaction):
"""Get all session for a project with PID"""
msg = transaction.asyncRead()
if "pid" in msg:
ret = []
uids = self.findSessionIDsByProjectId(msg["pid"])
for uid in uids:
session = self.findSessionBySessionUid(uid)
name = session.directory
name = os.path.basename(name)
if session.isConnected:
name = "(connected) " + name
state = session.state
if state is State.FINISHED:
name += " [FINISHED]"
if state is State.WAITING:
name += " [READY]"
if state is State.RUNNING:
name += " [RUNNING]"
if state is State.PAUSED:
name += " [PAUSED]"
if state is State.FAILED:
name += " [FAILED]"
ret.append((uid, name))
ret.sort(key=lambda s: s[1], reverse=True)
msg["sessions"] = ret
msg["status"] = True
logging.debug("Found %s Session/s", str(len(ret)))
else:
msg["status"] = False
msg["error"] = ["GetSessions: No PID provided"]
logging.error("Sessions not found. No PID provided.")
transaction.send(msg)
def findSessionBySessionUid(self, uid):
"""find a single session with UID"""
for sessiondict in self.sessions:
if sessiondict["uid"] == uid:
logging.debug("Found session for UID '%s'", uid)
return sessiondict["session"]
logging.debug("No sessions found for UID '%s'", uid)
return None
def findSessionIDsByProjectId(self, pid):
"""list all sessionids with PID"""
uids = []
for sessiondict in self.sessions:
if sessiondict["pid"] == pid:
uids.append(sessiondict["uid"])
logging.info("Found %s session/s for PID '%s'", str(len(uids)), pid)
return uids
def findSessionIDsWithState(self, state):
"""find all session UIDs with state"""
sessions = []
for sessiondict in self.sessions:
if sessiondict["session"].state == state:
sessions.append(sessiondict["uid"])
logging.debug("There are %i session/s with state %i", len(sessions), state)
return sessions
def _createSession(self, pid, dir):
"""Create a new session with PID in directory"""
# TODO
session = ServerSession(self, dir, pid=pid)
uid = session.uid
self.sessions.append({"pid": pid, "uid": uid, "session": session})
return uid
def _loadSession(self, dir):
"""load a session from directory"""
session = ServerSession(self, dir, parse_old=True)
return {"pid": session.pid, "uid": session.uid, "session": session}
def _createDirName(self, sid):
"""create a directory for session with SID"""
directory = self.__getDateTimeString()
directory += "_" + str(sid)
sdir = os.path.join(self.sessionPath, directory)
return sdir
def __getDateTimeString(self):
""" Return the current datetime in the format YYYYMMDD_hh24mmss
"""
now = datetime.now()
date_string = ''
date_string += str(now.year)
date_string += str(now.month).zfill(2)
date_string += str(now.day).zfill(2)
date_string += '_'
date_string += str(now.hour).zfill(2)
date_string += str(now.minute).zfill(2)
date_string += str(now.second).zfill(2)
return date_string
def loadSessions(self):
"""load all session from sessionpath and add them to the pool"""
pool = SessionPool()
count = 0
for entry in os.listdir(self.sessionPath):
if entry == "barista.conf" or entry == "caffeVersions":
continue
sdir = os.path.join(self.sessionPath, entry)
if self._isSession(sdir):
sessiondict = self._loadSession(sdir)
if sessiondict is not None:
self.sessions.append(sessiondict)
pool.addSession(sessiondict["session"])
count += 1
self.poolEmptyJob.connect(self.emptyJob)
pool.activate(lambda: self.poolEmptyJob.emit("All jobs finished. SessionPool Queue empty."))
logging.info("Loaded %s session/s", str(count))
return count
def emptyJob(self, msg):
logging.info("All jobs finished. SessionPool Queue empty.")
print msg
def _isSession(self, dir):
"""Check if directory contains a valid session."""
session_json = os.path.join(dir,Paths.FILE_NAME_SESSION_JSON)
if os.path.isdir(dir) is False:
sys.stderr.write("Session directory " + dir + " is invalid! No directory!\n")
logging.error("Session directory %s is invalid. No directory", dir)
return False
if not os.path.exists(session_json):
sys.stderr.write("Session directory " + os.path.basename(os.path.normpath(dir)) + " is invalid!\n File 'sessionstate.json' does not exist!\n")
logging.error("Session directory %s is invalid. 'sessionstate.json' does not exist!", dir)
return False
with open(session_json,"r") as file:
try:
dict = json.load(file)
except ValueError:
sys.stderr.write("Session file " + session_json + " is invalid!\n File 'sessionstate.json' could not be parsed!\n")
logging.error("Session file %s is invalid. 'sessionstate.json' could not be parsed!", session_json)
return False
for key in self.jsonKeys:
if key not in dict:
sys.stderr.write("Session directory "
+ os.path.basename(os.path.normpath(dir))+" is invalid!\n Key '"
+key +"' is missing in sessionstate!\n")
logging.error("Session at %s is invalid. Key %s in 'sessionstate.json' is missing.",dir,key)
return False
if "LastSnapshot" in dict:
if dict["LastSnapshot"]:
if not os.path.exists(os.path.join(dir,dict["LastSnapshot"])):
sys.stderr.write("Session directory "
+ os.path.basename(os.path.normpath(dir)) + " is invalid!\n Snapshot "
"was set but not found in directory\n")
logging.error("Session at %s is invalid. Snapshot was set but not found.", dir)
return False
logging.debug("Session directory %s is valid!", dir)
return True
def _parseSnapshotPrefixFromFile(self, filename):
""" Return the snapshot prefix of the solver.
"""
with open(filename) as f:
regex_prefix = re.compile(
'snapshot_prefix:[\s]+"(.+)"')
for line in f:
prefix_match = regex_prefix.search(line)
if prefix_match:
return prefix_match.group(1)
def isTraining(self):
"""check if there is already a session running on this server"""
return len(self.findSessionIDsWithState(State.RUNNING)) > 0
Barista/backend/networking/server_transaction.py 0000664 ڊ0011117 00000034750 13265130053 022440 0 ustar klemms o0cvpr import os
import sys
import logging
from PyQt5.QtCore import QFileInfo, QDirIterator, QDateTime, QRegExp, pyqtSignal
from PyQt5.QtCore import QTimer
from backend.networking.transaction import Transaction
from gui.input_manager.database_object import DatabaseObject
from backend.networking.protocol import Protocol
import backend.barista.caffe_versions as caffeVersions
import backend.barista.hash as Hash
import backend.barista.utils.db_util as db_util
class ServerTransaction(Transaction):
def __init__(self, parent):
self.options = {
Protocol.ECHO: self.msgEcho,
Protocol.RESTART: self.restart,
Protocol.GETDIR: self.getDir,
Protocol.GETSTATUS: self._getBaristaStatus,
Protocol.GETDBSTATUS: self._getDBStatus,
Protocol.MAKEPATHRELATIVE: self._makeRelative,
Protocol.MAKEHDF5TXT: self._makeHDF5Txt,
Protocol.ADDHDF5: self._addHdf5,
Protocol.GETFILECONTENT: self._getFileContent,
Protocol.WRITEFILECONTENT: self._writeFileContent,
Protocol.GETHASH: self._getHash,
Protocol.GETHARDWARE: self._getHardware,
Protocol.SCANHARDWARE: self._scanHardware,
Protocol.SETHARDWARE: self._setHardware,
Protocol.GETPLATFORM: self._getPlatform,
Protocol.CREATESESSION: self._createSession,
Protocol.CONNECTTOSESSION: self._connectToSession,
Protocol.GETSESSIONS: self._getSessions,
Protocol.DELETESESSION: self._deleteSession,
Protocol.SESSION: self._doNothing,
Protocol.CLONESESSION: self._cloneSession,
Protocol.DISCONNECTSESSION: self._disconnectSession,
Protocol.SESSION: self._doNothing,
Protocol.GETCAFFEVERSIONS: self._getCaffeVersions,
Protocol.ADDCAFFEVERSION: self._addCaffeVersion,
Protocol.SETCURRENTCAFFEVERSION: self._setCurrentCaffeVersion,
Protocol.REMOVECAFFEVERSION: self._removeCaffeVersion,
Protocol.GETDEFAULTCAFFEVERSION: self._getDefaultCaffeVersion,
Protocol.GETCAFFERESTART: self._getCaffeRestart,
Protocol.GETFILEHASH: self._getFileHash,
Protocol.GETDIRHASH: self._getDirHash
}
Transaction.__init__(self)
self.parent = parent
def processMessage(self):
"""process the received message and call further processing"""
# key = self.messageOutput[0]["key"]
key = self.getAttrOfFirst(["key"])[0] # Threadsafe
if key in self.options:
self.options[key]()
else:
sys.stderr.write("Wrong Key: " + str(key) + "\n")
def msgEcho(self):
"""return the message back to client. this is for testing."""
msg = self.asyncRead()
self.send(msg)
def restart(self):
"""Saves all server-sessions of the current project and restarts the host afterwards."""
msg = self.asyncRead()
if msg and "pid" in msg:
path = msg["pid"]
sessionsUIDs = self.parent.sessionManager.findSessionIDsByProjectId(msg["pid"])
success = True
for uid in sessionsUIDs:
session = self.parent.sessionManager.findSessionBySessionUid(uid)
if session.isConnected:
if not session.save():
success = False
if success is False:
msg["error"] = ["Could not save a session because it contains layer of another caffe-version."]
msg["status"] = False
self.send(msg)
else:
os.execl(sys.executable, sys.executable, *sys.argv)
def getDir(self):
"""list all files and subdirectories for a given dir with regex."""
msg = self.asyncRead()
if msg:
path = msg["path"]
dirselect = False
if "dirSelect" in msg:
dirselect = msg["dirSelect"]
if path == "":
path = "/home"
path = os.path.normpath(path)
filters = ["*"]
if "filter" in msg:
filters = msg["filter"]
dirIter = QDirIterator(path)
data = list()
while dirIter.hasNext():
dirIter.next()
if dirIter.fileName() in [".", ".."]:
continue
if not dirIter.fileInfo().isDir():
if dirselect:
continue
match = False
for fi in filters:
reg = QRegExp(fi)
reg.setPatternSyntax(QRegExp.Wildcard)
if reg.exactMatch(dirIter.fileName()):
match = True
break
if not match:
continue
elem = dict()
elem["name"] = dirIter.fileName()
elem["path"] = dirIter.filePath()
inf = dirIter.fileInfo() # type: QFileInfo
elem["isDir"] = inf.isDir()
last = inf.lastModified() # type: QDateTime
elem["lastChange"] = last.toString("yyyy.MM.dd - HH:MM")
if inf.isDir():
elem["fileSize"] = ""
else:
elem["fileSize"] = self._sizeHumanReadableStr(inf.size())
data.append(elem)
data = sorted(data, key=lambda di: (not di["isDir"], di["name"].lower()))
msg["data"] = data
logging.debug("There is/are %i element/s in %s", len(data), path)
self.send(msg)
def _sizeHumanReadableStr(self, size):
"""convert integer to human readable size"""
sizes = ["B", "kB", "MB", "GB", "TB"]
curSize = 0
unit = sizes[curSize]
while (size > 1000 and curSize < len(sizes) - 1):
curSize = curSize + 1
unit = sizes[curSize]
size = round(size / 1000.0, 1)
return str(size) + " " + unit
def _getBaristaStatus(self):
"""start the Timer for Heartbeat"""
msg = self.asyncRead()
if "projectid" in msg.keys():
self.projectID = msg["projectid"]
if not hasattr(self, "timer"):
self.timer = QTimer()
self.timer.setInterval(5000)
self.timer.setSingleShot(True)
self.timer.timeout.connect(self._sendBaristaStatus)
self.socketClosed.connect(lambda : self.timer.stop())
logging.debug("start heartbeat timer")
self._sendBaristaStatus()
def _sendBaristaStatus(self):
"""return the server status"""
if hasattr(self, "timer"):
self.timer.stop()
if self.isConnected():
self.timer.start()
msg = {"key": Protocol.GETSTATUS}
if hasattr(self, "projectID"):
state = self.parent.getBaristaStatus(self.projectID)
else:
state = self.parent.getBaristaStatus()
msg["data"] = state
logging.debug("got state")
self.send(msg)
def _getDBStatus(self):
"""open a database, check for status and get properties"""
msg = self.asyncRead()
dbo = DatabaseObject()
path = msg["path"]
if "absolute" not in msg:
path = os.path.join(self.parent.sessionPath, path)
dbo.setDB(path, msg["type"])
if msg["type"] == "HDF5TXT":
dbo.setProjectPath(self.parent.sessionPath)
dbo.open()
status = dbo.isOpen()
msg["status"] = status
if status:
msg["dataCount"] = dbo.getDataCount()
msg["dimensions"] = dbo.getDimensions()
dbo.close()
logging.debug("Database: %s, Status: %s", path, str(status))
self.send(msg)
def _makeRelative(self):
"""make a given path relative to sessionpath"""
msg = self.asyncRead()
path = msg["path"]
sesPath = self.parent.sessionPath
msg["path"] = os.path.relpath(path, sesPath)
logging.debug("old: '%s', new: '%s'", path, msg["path"])
self.send(msg)
def _makeHDF5Txt(self):
"""create the hdf5txt file"""
msg = self.asyncRead()
msg["status"] = False
path = msg["path"]
if not os.path.exists(path):
with open(path, 'w') as file:
file.write(msg["hdf5"])
msg["status"] = True
logging.debug("File created: %s", str(msg["status"]))
self.send(msg)
def _addHdf5(self):
"""add a line to HDF5txt"""
msg = self.asyncRead()
msg["status"] = False
path = os.path.join(self.parent.sessionPath, msg["path"])
if os.path.exists(path):
with open(path, 'a') as file:
file.write("\n" + msg["hdf5"])
msg["status"] = True
logging.debug("Line written: %s", str(msg["status"]))
self.send(msg)
def _getFileContent(self):
"""read a file and return contents"""
# TODO this can be a security flaw since it allows to read arbitrary files.
msg = self.asyncRead()
msg["status"] = False
path = os.path.join(self.parent.sessionPath, msg["path"])
if os.path.exists(path):
with open(path, 'r') as file:
msg["file"] = file.read()
msg["status"] = True
else:
logging.warning("File does not exist: %s", path)
self.send(msg)
def _writeFileContent(self):
"""write a file"""
# TODO this can be a security flaw since it allows to write arbitrary files.
msg = self.asyncRead()
msg["status"] = False
path = os.path.join(self.parent.sessionPath, msg["path"])
if os.path.exists(path):
with open(path, 'w') as file:
file.write(msg["file"])
msg["status"] = True
else:
logging.warning("File does not exist: %s", path)
self.send(msg)
"""
Computes and sends the hashValue of a db file, if this file exists.
@author : j_stru18
"""
def _getHash(self):
msg = self.asyncRead()
msg["status"] = False
if "path" in msg and "type" in msg:
path = msg["path"]
type = msg["type"]
hashValue = 0
if path is not None and type is not None:
if os.path.exists(path):
dbPaths = db_util.getDBPathsByType(path, type)
hashValue = db_util.getMultipleHash(dbPaths)
msg["status"] = True
else:
logging.warning("File does not exist: %s", path)
else:
logging.warning("Internal error: Remote message gave no path and type.")
msg["hashValue"] = hashValue
else:
logging.warning("Internal error: Remote message was faulty.")
self.send(msg)
def _getHardware(self):
"""get the hardware specs"""
msg = self.asyncRead()
msg["hardware"] = self.parent.hardware
msg["current"] = self.parent.trainOnHW
self.send(msg)
def _scanHardware(self):
"""launch a hardware scan"""
self.parent.checkHardware(True, self)
def _setHardware(self):
msg = self.asyncRead()
status = self.parent.setHardware(msg["hid"])
msg["status"] = status
logging.info("Set current hardware to HID %i: %s", msg["hid"], str(status))
self.send(msg)
def _getPlatform(self):
msg = self.asyncRead()
msg["platform"] = sys.platform
self.send(msg)
def _createSession(self):
self.parent.sessionManager.createNewSession(self)
def _cloneSession(self):
self.parent.sessionManager.cloneSession(self)
def _disconnectSession(self):
self.parent.sessionManager.disconnectSession(self)
def _deleteSession(self):
self.parent.sessionManager.deleteSession(self)
def _connectToSession(self):
self.parent.sessionManager.connectToSession(self)
def _getSessions(self):
self.parent.sessionManager.getSessions(self)
def _doNothing(self):
pass
def _getCaffeVersions(self):
versions = caffeVersions.getAvailableVersions()
msg = self.asyncRead()
versionsToSend = {}
for version in versions:
versionsToSend[version.getName()] = {"root": version.getRootpath(),
"binary": version.getBinarypath(),
"python": version.getPythonpath(),
"proto": version.getProtopath()}
msg["versions"] = versionsToSend
msg["status"] = True
self.send(msg)
def _addCaffeVersion(self):
msg = self.asyncRead()
versionReceived = msg["version"]
version = caffeVersions.caffeVersion(versionReceived["name"], versionReceived["root"], versionReceived["binary"], versionReceived["python"], versionReceived["proto"])
caffeVersions.addVersion(version, self.parent.sessionPath)
msg["status"] = True
self.send(msg)
def _setCurrentCaffeVersion(self):
msg = self.asyncRead()
versionNameReceived = msg["versionname"]
caffeVersions.setDefaultVersion(versionNameReceived, self.parent.sessionPath)
caffeVersions.restart = True
msg["status"] = True
self.send(msg)
def _removeCaffeVersion(self):
msg = self.asyncRead()
versionNameReceived = msg["versionname"]
version = caffeVersions.getVersionByName(versionNameReceived)
caffeVersions.removeVersion(version, self.parent.sessionPath)
msg["status"] = True
self.send(msg)
def _getDefaultCaffeVersion(self):
msg = self.asyncRead()
if caffeVersions.getDefaultVersion():
msg["defaultVersionName"] = caffeVersions.getDefaultVersion().getName()
msg["status"] = True
else:
msg["status"] = False
self.send(msg)
def _getCaffeRestart(self):
msg = self.asyncRead()
msg["cafferestart"] = caffeVersions.restart
msg["status"] = True
self.send(msg)
def _getFileHash(self):
msg = self.asyncRead()
path = msg["path"]
hash = Hash.hashFile(path)
msg["status"] = True
msg["hash"] = hash
self.send(msg)
def _getDirHash(self):
msg = self.asyncRead()
path = msg["path"]
hash = Hash.hashDir(path)
msg["status"] = True
msg["hash"] = hash
self.send(msg) Barista/backend/networking/transaction.py 0000664 ڊ0011117 00000031754 13265130053 021053 0 ustar klemms o0cvpr import pickle
import sys
from abc import abstractmethod
from threading import Lock
import logging
import zlib
from PyQt5.QtCore import QByteArray, QDataStream, QIODevice, QObject, pyqtSignal
from PyQt5.QtNetwork import QTcpSocket, QAbstractSocket
from PyQt5.QtTest import QSignalSpy
from PyQt5.QtWidgets import qApp
COMPRESS = True
class Transaction(QObject):
"""Use parent class for server and client transactions. Abstracts TCPSocket-Access"""
# empirical benchmarks using the echo command (send + recieve) shows following performance characteristics:
# - a server can handle thousands of messages on one open socket per second, more CPU performance --> faster server
# - as long as you don't saturate your network connection by increasing the message size there
# should be no problems at all
# - increasing the message size beyond the saturation of the network connection may lead to message drops
# and crash the server
#
# some examples:
# - a server on a laptop connected over a 2-3 mb/s wifi would crash if the message size is greater then aprox 6mb
# - the same laptop with an ethernet cable can handle up to aprox 20 mb per message
# - a server over localhost and the loopback interface could handle up to aprox 100mb
# TODO longer testing
# TODO close and reopen the socket after each / after n messages
# TODO investigate message drops and server crashed
# TODO maybe rewrite the class to split messages over certain size
# splitting messages refers to writing at most n bytes to the QDataStream and flushing regulary
# alternatively split the message altogether
# this depends if the QTcpSocket or the QDataStream is the bottleneck
bufferReady = pyqtSignal()
socketClosed = pyqtSignal()
_stopWaiting = pyqtSignal()
_staging = pyqtSignal()
def __init__(self):
# QObject.__init__(self)
super(Transaction, self).__init__()
self.tcpsocket = None # type: QTcpSocket
self.messageBuffer = QByteArray()
self.recieveBuffer = QByteArray()
self.messageOutput = list()
self.messageSize = 0
self.isRecieving = False
self.bufferReady.connect(self.processMessage)
self._isConnected = False
self._hasError = False
self.lock = Lock()
def connect(self, host, port):
"""connect to host on port"""
logging.debug("Connecting to %s:%i", host, port)
self._hasError = False
self.tcpsocket = QTcpSocket()
self.tcpsocket.error.connect(self.processError)
self.tcpsocket.connected.connect(self._connected)
self.tcpsocket.connected.connect(lambda: self._stopWaiting.emit())
self.tcpsocket.readyRead.connect(self.receive)
self.tcpsocket.connectToHost(host, port)
self.waitForConnection()
def processError(self, error):
"""
Process the errors on the tcpsocket
See: http://doc.qt.io/qt-5/qabstractsocket.html#SocketError-enum
"""
logging.debug("Error %i", error)
if error == 0:
# Remote Host can't be resolved
self.close()
elif error == 1:
# Remote Host closed the connection.
self.close()
else:
# TODO: somehow handle errors
# Code 7 is detected far too late, it waits till the socket itself times out
errortxt = ""
for k in dir(QAbstractSocket):
v = getattr(QAbstractSocket, k)
if type(v) is type(error) and str(v) is str(error):
errortxt = k
sys.stderr.write("QTcpSocket@Transaction Error Code: " + str(error) + " " + errortxt + "\n")
self.close()
self._isConnected = False
self._hasError = True
self._stopWaiting.emit()
def _connected(self):
"""On Connect mark this transactions as connected"""
logging.debug("Socket connected.")
self._isConnected = True
self._hasError = False
def acceptClient(self, socket):
"""set external socket"""
logging.debug("Client socket connected.")
self.tcpsocket = socket
self._isConnected = True
self._hasError = False
self.tcpsocket.error.connect(self.processError)
self.tcpsocket.readyRead.connect(self.receive)
def receive(self):
"""On recieve packet load message into buffer. But first get message length"""
logging.debug("Receiving message")
while self.tcpsocket.bytesAvailable() > 0:
# while some unread bytes available
stream = QDataStream(self.tcpsocket)
stream.setVersion(QDataStream.Qt_5_3)
if not self.isRecieving: # For a new Message get the Message Size
if self.tcpsocket.bytesAvailable >= 4: # since reading UInt32
self.messageSize = stream.readUInt32()
logging.debug("Start of new message of size %i", self.messageSize)
self.isRecieving = True
else:
break
else: # For a continued message keep reding until whole message is in buffer
s = min(self.tcpsocket.bytesAvailable(), self.messageSize - len(self.messageBuffer))
self.messageBuffer.append(stream.readRawData(s))
if len(self.messageBuffer) == self.messageSize:
logging.debug("Finished receiving message of size %i", self.messageSize)
self._processBuffer()
def send(self, msg):
"""Send first the message size, then the message in Pickle"""
if self.isConnected():
pmsg = pickle.dumps(msg)
if COMPRESS:
pmsg = zlib.compress(pmsg)
buffer = QByteArray()
stream = QDataStream(buffer, QIODevice.WriteOnly)
stream.setVersion(QDataStream.Qt_5_3)
stream.writeUInt32(len(pmsg))
stream.writeRawData(pmsg)
bytesWritten = self.tcpsocket.write(buffer)
self.tcpsocket.flush()
self.tcpsocket.waitForBytesWritten()
# qApp.processEvents() # send data immediately and don't wait for next mainloop
logging.debug("Bytes written: %i", bytesWritten)
if bytesWritten > 0:
return True
else:
logging.debug("Message not send. Not connected")
return False
def isConnected(self):
"""return the connected state"""
return self._isConnected
def close(self):
"""close this socket and fire signal socketClosed to notify slots to delete this obj"""
if self.tcpsocket:
logging.debug("socket closing")
self._isConnected = False
self.tcpsocket.close()
# self.tcpsocket = None
self.socketClosed.emit()
def _processBuffer(self):
"""If message is complete: read Pickle and move to output buffer"""
pmsg = str(self.messageBuffer)
if COMPRESS:
pmsg = zlib.decompress(pmsg)
msg = pickle.loads(pmsg)
logging.debug("Message received: %s", str(msg))
self.lock.acquire()
self.messageOutput.append(msg)
self.lock.release()
# reset state
self.isRecieving = False
self.messageBuffer.clear()
self._stopWaiting.emit()
self.bufferReady.emit()
@abstractmethod
def processMessage(self):
"""implement in server/client"""
pass
def asyncRead(self, timeout=5000, staging=False, attr=None):
"""For direct reading access, pop the first message in buffer
Explanation:
timeout: stop after waiting for 5s and return NONE object.
this is usefull if the server does not answer (e.g. crash)
or the networkconnection breaks
staging: If True the asynchronous wait will wait for a the staging signal instead of the messageOutput signal.
the staging signal should be used if you have a class which uses both: a signal/slot based processMessage
methode and asyncRead calls. This gives the processMessage calls priority.
!!!Don't forget to call the stage() method if your processMessage ignores the message!!!
attr: In some cases you have multiple asyncRead calls on the same transaction Item.
This may lead to preemtive behaviour resulting in reading the wrong messages and key errors.
(e.g. READ#1 reads messages destined for READ#2)
By providing a tupel (attr, value) the message is parsed and only poped if the value matches.
Not matching messages are appended back into the buffer and other waiting asyncReads are notified
"""
if self.isConnected():
turns = 0
while True:
turns += 1
if turns > 100:
logging.debug("Timeout on read after 100 iterations")
return None
result = True
logging.debug("MessageOutput size: %i", len(self.messageOutput))
if len(self.messageOutput) == 0 and not self.containsAttr(attr):
logging.debug("Waiting for new message.")
if not staging:
# spy = QSignalSpy(self._stopWaiting)
spy = QSignalSpy(self.bufferReady)
else:
spy = QSignalSpy(self._staging)
result = spy.wait(timeout) # Asynchronous wait, Timeout 5s
if result and not self._hasError:
self.lock.acquire()
if len(self.messageOutput) == 0:
self.lock.release()
logging.debug("Race condition triggered. Wait for next message.")
continue
found = False
result = self.messageOutput[0]
if attr is not None:
for msg in self.messageOutput:
if attr[0] in msg:
if attr[1] == msg[attr[0]]:
found = True
result = msg
break
if found or attr is None:
del self.messageOutput[self.messageOutput.index(result)]
logging.debug("MessageOutput size: %i", len(self.messageOutput))
self.lock.release()
if "error" not in result:
result["error"] = []
if "status" not in result:
result["status"] = True
return result
else:
logging.debug("Message not found. Release of lock.")
if attr is not None:
logging.debug("Miss '%s' with value '%s'", str(attr[0]), str(attr[1]))
self.lock.release()
self.bufferReady.emit()
qApp.processEvents()
else:
logging.debug("Nothing to read.")
break
else:
logging.debug("Not connected. Did not read.")
return None
def stage(self):
logging.debug("Stage!")
self._staging.emit()
def waitForConnection(self):
"""sometimes it takes some time to establish the connection. wait half a second"""
logging.debug("Establishing connection")
if self.tcpsocket is not None:
result = True
if not self.isConnected() and not self._hasError:
# spy = QSignalSpy(self.tcpsocket.connected)
spy = QSignalSpy(self._stopWaiting)
result = spy.wait(5000) # Asynchronous wait, Timeout 5 s
if not result:
# it is bad if the socket needs longer than the timeout and connects
# after the connection is deemed dead
self.close()
logging.debug("Connection not established, manually closing socket.")
return result and not self._hasError
return False
def getAttrOfFirst(self, attrlist):
"""check the attributes of the first item in the output buffer"""
self.lock.acquire()
result = []
for attr in attrlist:
if attr in self.messageOutput[0]:
result.append(self.messageOutput[0][attr])
else:
result.append(None)
self.lock.release()
return result
def containsAttr(self, attr):
"""check if any message in the output buffer contains the desired attribute"""
if attr is None:
return False
self.lock.acquire()
found = False
for msg in self.messageOutput:
if attr[0] in msg:
if msg[attr[0]] is attr[1]:
found = True
break
self.lock.release()
return found
Barista/backend/parser/__init__.py 0000664 ڊ0011117 00000000000 13265130053 017346 0 ustar klemms o0cvpr Barista/backend/parser/concatenator.py 0000664 ڊ0011117 00000020074 13265130053 020304 0 ustar klemms o0cvpr '''
Created on 18.11.2016
@author: k.Hartz
'''
import itertools
import os
import re
from backend.parser.parser_listener import ParserListener
from backend.barista.utils.logger import Log
from backend.parser.parser import Parser
class Concatenator:
'''
Concatenates multiple logfiles (with same snapshot-file) to one iterator
'''
def __init__(self, listOfLogfiles):
'''
Initializes the concatinator with a list of Logfiles
to get a list of iterators concerning to the merged logs
call concate()
'''
self.__listeners = []
events = {
Concatenator.finish: re.compile('Optimization Done.'),
Concatenator.interrupt: re.compile(
"((?:Snapshotting solver state to (?:binary proto|HDF5) file ).*)"),
Concatenator.resume: re.compile("((?<= Resuming from ).*)"),
}
for logFile in listOfLogfiles:
log_iter = self.__createIterFromFile(logFile)
parser = Parser(log_iter, events)
listener = Concatenator.SnapshotListener(logFile)
parser.addListener(listener)
try:
parser.parseLog()
self.__listeners.append(listener)
except:
callerId = Log.getCallerId("Error Parsing Log File:" + str(logFile))
Log.error("", callerId)
def createSingleLogfile(self, filePath, logfileName, stream=None):
'''
creates one logfile for each Training
returns amount of created files or -1 for none
'''
if(stream is not None):
fullIter = [self.concateWithStream(stream)]
else:
fullIter = self.concate()
i = -1
for itera in fullIter:
i += 1
with open(filePath + logfileName + str(i) + '.log', 'w') as f:
for line in itera:
f.write(str(line))
return i+1
# TODO: Test
def concateWithStream(self, stream):
'''
concatenates (only!) the first (concatenated) logfile
with the given stream to an iterator, first the logfiles
than the stream
'''
return itertools.chain(iter(self.concate()[0]), stream)
def concate(self):
'''
concatenate the logfiles into one iterator
returns an array of concatenated iterators
every iterator represents one training
'''
if(len(self.__listeners) == 1):
return [self.__listeners[0].filePath]
listenerRange = range(0, len(self.__listeners))
for i in listenerRange:
for j in listenerRange: # TODO can start from i ... length
if(i != j and (self.__listeners[i].groupId == -1 or self.__listeners[j].groupId == -1 ) and
self.__fromSameTraining(i, j)):
self.__setGroupId(i, j)
for i in listenerRange:
if(self.__listeners[i].groupId == -1):
self.__setGroupId(i, i)
listenerGroups = self.__groupListeners()
return self.__createAndConcateIters(listenerGroups)
# Training Interrupted, snapshot is made
__groupIdCounter = 0
# ------EVENT CONSTANTS-----
# Training Completet (Caffe exited)
finish = 'Finished'
# Training Resumed from snapshot
resume = 'Resume'
# Training Interrupted, snapshot is made
interrupt = 'Interrupted'
class SnapshotListener(ParserListener):
'''
Listener for Logfile-Parser, looks for properties
that are needed to determine which logs should
concate
'''
def __init__(self, logfilePath):
# id to put listeners together
self.groupId = -1
# to be able to create an iter from this listener
self.filePath = logfilePath
# to identify the listeners ancestor
self.logname = self.__getFileName(logfilePath)
# has finished parsing
self.finish = False
# has been resumed from another session
self.resume = False
# hasSnapshot
self.hasSnapshot = False
self.snapshotName = ""
# the name of the resumed snapshot file
self.snapshotResumeName = ""
# start iteration count
self.iterationMin = -1
# finished iteration count
self.iterationMax = -1
def update(self, phase, row):
if phase == "TRAIN":
iterations = row.get('NumIters')
if iterations > self.iterationMax:
self.iterationMax = int(iterations)
if self.iterationMin == -1:
self.iterationMin = int(iterations)
def handle(self, event, message, groups):
'''
handles events: finish,interrupt,resume
'''
if(event == Concatenator.finish):
self.finish = True
elif(event == Concatenator.interrupt):
self.hasSnapshot = True
self.snapshotName = groups[0].strip()
elif(event == Concatenator.resume):
self.hasSnapshot = True
self.snapshotResumeName = groups[0].strip()
def __getFileName(self, filePath):
'''
extracts the filename from a filepath
'''
_, tail = os.path.split(str(filePath))
return tail
# ----------Private Methods-------------
def __fromSameTraining(self, i, j):
'''
returns boolean indicating whether index i and j
are from the same training
'''
iListener = self.__listeners[i]
jListener = self.__listeners[j]
# Merge every log
if(False):
return True
# have snapshots?
if not(iListener.hasSnapshot and
jListener.hasSnapshot):
return False
# Same snapshotname?
if(iListener.snapshotName != jListener.snapshotResumeName and
jListener.snapshotName != iListener.snapshotResumeName):
return False
range1 = range(iListener.iterationMin,
iListener.iterationMax)
range2 = range(jListener.iterationMin,
jListener.iterationMax)
# Iterations doesnt have intersect
if(len(set(range1).intersection(range2)) > 1):
return False
return True
def __setGroupId(self, i, j):
'''
Sets the same group id for index i,j in
self.__listeners
'''
if self.__listeners[i].groupId != -1:
self.__listeners[j].groupId = self.__listeners[i].groupId
return
if self.__listeners[i].groupId != -1:
self.__listeners[j].groupId = self.__listeners[i].groupId
return
Concatenator.__groupIdCounter += 1
self.__listeners[j].groupId = Concatenator.__groupIdCounter
self.__listeners[j].groupId = Concatenator.__groupIdCounter
def __groupListeners(self):
'''
groups the listeners by id and sorts them
by the startiterations
'''
self.__listeners.sort(key=lambda listener:
(listener.groupId, listener.iterationMin))
listenerGroups = itertools.groupby(self.__listeners,
lambda listener: (listener.groupId))
sortedList = []
for _, group in listenerGroups:
group = list(group)
sortedList.append(group)
return sortedList
def __createAndConcateIters(self, listenerGroups):
'''
creates an iterator for every
'''
iterators = []
for group in listenerGroups:
for listener in group:
itera = listener.filePath
iterators.append(itera)
# for i in range(0, len(iterators)):
# iterators[i] = fileinput.input(iterators[i])
return iterators
def __createIterFromFile(self, filePath):
'''
creates an iterator for a logfile
for a given path
'''
return (open(filePath))
Barista/backend/parser/dummy_std_out.py 0000664 ڊ0011117 00000001324 13265130053 020515 0 ustar klemms o0cvpr '''
Created on 08.11.2016
@author: k.hartz
'''
import os
import time
import sys
# Logfile to pass through stdout
logFile = "test/logfiles/test_log_1.log"
# delay per line in sec, 1=1sec
delayPerLine = 0.01
# Seperator to seperate the logfile output from print
seperator = "------------------"
# directory where this file is contained
dir_path = os.path.dirname(os.path.realpath(__file__))
# working directory
cwd = os.getcwd()
print("Start Output:\n\n")
print("working directory:", cwd)
print("directory where this file is contained:", dir_path)
print(seperator)
f = open(logFile)
for line in f:
sys.stdout.write(line)
sys.stdout.flush()
time.sleep(delayPerLine)
print(seperator)
print("Finished log")
Barista/backend/parser/extract_seconds.py 0000775 ڊ0011117 00000007151 13265130053 021020 0 ustar klemms o0cvpr """
COPYRIGHT
All contributions by the University of California:
Copyright (c) 2014, 2015, The Regents of the University of California (Regents)
All rights reserved.
All other contributions:
Copyright (c) 2014, 2015, the respective contributors
All rights reserved.
Caffe uses a shared copyright model: each contributor holds copyright over
their contributions to Caffe. The project versioning records all such
contribution and copyright details. If a contributor wants to further mark
their specific copyright on a particular contribution, they should indicate
their copyright solely in the commit message of the change when it is
committed.
LICENSE
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:
1. Redistributions of source code must retain the above copyright notice, this
list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright notice,
this list of conditions and the following disclaimer in the documentation
and/or other materials provided with the distribution.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
CONTRIBUTION AGREEMENT
By contributing to the BVLC/caffe repository through pull-request, comment,
or otherwise, the contributor releases their content to the
license and copyright terms herein.
"""
import re
from datetime import datetime
def extractDatetimeFromLine(line, year):
"""Extract the datetime from the line.
Returns the parsed datetime with the added year.
"""
# Expected format:
# I0210 13:39:22.381027 2521 solver.cpp:204]
line = line.strip().split()
month = int(line[0][1:3])
day = int(line[0][3:])
timestamp = line[1]
pos = timestamp.rfind('.')
ts = [int(x) for x in timestamp[:pos].split(':')]
hour = ts[0]
minute = ts[1]
second = ts[2]
microsecond = int(timestamp[pos + 1:])
dt = datetime(year, month, day, hour, minute, second, microsecond)
return dt
def getLogCreatedYear(input_file):
"""Get year from log file start time.
Search for 'Log file created at:'
Use current year if the string can not be found.
"""
regex_year = re.compile('Log file created at: ([\d]{4})')
for line in input_file:
year_match = regex_year.search(line)
if year_match:
return int(year_match.group(1))
if isLogFormat(line):
now = datetime.now()
return now.year
def getStartTime(line, year):
"""Find start time in the line.
"""
if line.find('Solving') != -1:
return extractDatetimeFromLine(line, year)
regex_log_msg = re.compile(
'[IWEF][\d]{4} [\d]{2}:[\d]{2}:[\d]{2}\.[\d]{6}[\s]+[\d]+[\s]+[\w\.]+:[\d]+\]')
def isLogFormat(line):
""" Return True if the line has the log format.
"""
log_match = regex_log_msg.search(line)
if log_match:
return True
else:
return False
Barista/backend/parser/parser.py 0000664 ڊ0011117 00000027700 13265130053 017123 0 ustar klemms o0cvpr """
COPYRIGHT
All contributions by the University of California:
Copyright (c) 2014, 2015, The Regents of the University of California (Regents)
All rights reserved.
All other contributions:
Copyright (c) 2014, 2015, the respective contributors
All rights reserved.
Caffe uses a shared copyright model: each contributor holds copyright over
their contributions to Caffe. The project versioning records all such
contribution and copyright details. If a contributor wants to further mark
their specific copyright on a particular contribution, they should indicate
their copyright solely in the commit message of the change when it is
committed.
LICENSE
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:
1. Redistributions of source code must retain the above copyright notice, this
list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright notice,
this list of conditions and the following disclaimer in the documentation
and/or other materials provided with the distribution.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
CONTRIBUTION AGREEMENT
By contributing to the BVLC/caffe repository through pull-request, comment,
or otherwise, the contributor releases their content to the
license and copyright terms herein.
"""
import re
import sys
if sys.version_info[0] == 2:
from Queue import Queue
else:
from queue import Queue
from collections import OrderedDict
from threading import Lock
from backend.parser import extract_seconds as exs
from backend.barista.utils.logger import Log
from backend.parser.parser_common import ParserCommon
from PyQt5.QtCore import pyqtSignal
from PyQt5.Qt import QObject
class Parser(QObject, ParserCommon):
""" Log File Parser.
This class takes a stream of log messages and parses them on parseLog().
A listener could be registered for parsing events like log records,
exceptions and new keys.
"""
printLogSignl = pyqtSignal(str, bool)
def __init__(self, log_stream, events, log_id=None):
super(Parser, self).__init__()
self.train_dict_list = []
self.test_dict_list = []
self.log_stream = log_stream
self.thread_id = None
self.start_time = None
self.log_id = log_id
self.logging = False
self.streams = Queue()
self.lock = Lock()
if self.log_stream:
self.streams.put(self.log_stream)
self.events = events
if self.events is None:
self.events = {}
def printLog(self, log, error=False):
if self.log_id is not None:
if not error:
Log.log(log, self.log_id)
else:
Log.error(log, self.log_id)
else:
self.printLogSignl.emit(log, error)
def getTestEvents(self):
"""Return the list of parsed test records."""
return self.test_dict_list
def getTrainEvents(self):
"""Return the list of parsed training records."""
return self.train_dict_list
def getThreadID(self):
""" Return the thread id of the caffe process.
"""
return self.thread_id
def addLogStream(self, log_stream):
""" Add a log stream to to queue of parsing tasks.
"""
if log_stream:
self.streams.put(log_stream)
def setLogging(self, log):
self.logging = log
def parseLog(self):
"""Parse log file.
Returns (train_dict_list, test_dict_list)
train_dict_list and test_dict_list are lists of dicts that define the
table rows
"""
locked = self.lock.acquire()
if locked is False:
return self.train_dict_list, self.test_dict_list
regex_iteration = re.compile('Iteration (\d+)')
regex_train_output = re.compile(
'Train net output #(\d+): (\S+) = ([\.\deE+-]+)')
regex_test_output = re.compile(
'Test net output #(\d+): (\S+) = ([\.\deE+-]+)')
regex_learning_rate = re.compile(
'lr = ([-+]?[0-9]*\.?[0-9]+([eE]?[-+]?[0-9]+)?)')
# Pick out lines of interest
iteration = -1
learning_rate = float('NaN')
train_dict_list = self.train_dict_list
test_dict_list = self.test_dict_list
train_row = None
test_row = None
try:
while self.streams.empty() is False:
head = self.streams.get()
log_file = None
ht = type(head)
if ht is unicode or ht is str:
# open file
log_file = open(head, 'r')
log_stream = log_file.readlines()
else:
log_stream = head
try:
logfile_year = exs.getLogCreatedYear(log_stream)
for line in log_stream:
line = line.strip()
if self.logging:
self.printLog(line)
if self.thread_id is None:
self.extractThreadID(line)
if self.start_time is None:
self.start_time = exs.getStartTime(line,
logfile_year)
self.parseEvent(line)
iteration_match = regex_iteration.search(line)
if iteration_match:
iteration = float(iteration_match.group(1))
if iteration == -1:
# Only start parsing for other stuff if we've
# found the first iteration
continue
if exs.isLogFormat(line) is False:
continue
if self.start_time is None:
self.start_time = exs.extractDatetimeFromLine(line,
logfile_year)
time = exs.extractDatetimeFromLine(line,
logfile_year)
seconds = (time - self.start_time).total_seconds()
lr_match = regex_learning_rate.search(line)
if lr_match:
self.checkKey(Parser.TRAIN, 'LearningRate')
learning_rate = float(lr_match.group(1))
train_dict_list, train_row = self.parseLine(
regex_train_output, train_row, train_dict_list,
line, iteration, seconds, learning_rate, time,
Parser.TRAIN
)
test_dict_list, test_row = self.parseLine(
regex_test_output, test_row, test_dict_list,
line, iteration, seconds, learning_rate, time,
Parser.TEST
)
except Exception as e:
self.printLog('Parser error: '+str(e))
finally:
if log_file:
log_file.close()
except Exception as e:
if self.log_id:
self.printLog('Failed to parse log '+str(e), True)
else:
print('Failed to parse log '+str(e))
finally:
if locked:
self.lock.release()
for lis in self.listener:
lis.parsingFinished()
return train_dict_list, test_dict_list
def parseLine(self, regex_obj, row, row_dict_list, line, iteration,
seconds, learning_rate, time, phase):
"""Parse a single line for training or test output.
Returns a a tuple with (row_dict_list, row)
row: may be either a new row or an augmented version of the current row
row_dict_list: may be either the current row_dict_list or an augmented
version of the current row_dict_list
"""
output_match = regex_obj.search(line)
if output_match:
if not row or row['NumIters'] != iteration:
# Push the last row and start a new one
if row:
# If we're on a new iteration, push the last row
# This will probably only happen for the first row;
# otherwisethe full row checking logic below will push and
# clear full rows
row_dict_list.append(row)
row = OrderedDict([
('NumIters', iteration),
('Seconds', seconds),
('LearningRate', learning_rate),
('DateTime', time)
])
# output_num is not used; may be used in the future
# output_num = output_match.group(1)
output_name = output_match.group(2)
output_val = output_match.group(3)
row[output_name] = float(output_val)
self.checkKey(phase, output_name)
# append a new row to the dict
if (row and
len(row_dict_list) >= 1 and
len(row) == len(row_dict_list[0])):
# fix the learning rate of the first row
if len(row_dict_list) == 1:
row_dict_list[0]['LearningRate'] = row['LearningRate']
for lis in self.listener:
lis.update(phase, row_dict_list[0])
# The row is full, based on the fact that it has the same number of
# columns as the first row; append it to the list
row_dict_list.append(row)
# notify the listener about the new row
for lis in self.listener:
lis.update(phase, row)
row = None
return row_dict_list, row
def parseEvent(self, line):
"""Parse the line for events.
Notifies the listener about every event found.
"""
for event, regex in self.events.iteritems():
event_match = regex.search(line)
if event_match:
size = len(event_match.groups())
groups = []
for i in range(0, size):
groups.append(event_match.group(i+1))
for lis in self.listener:
lis.handle(event, line, groups)
def extractThreadID(self, line):
""" Extract the thread id from the file.
"""
regex_thread_id = re.compile(
'[IWEF][\d]{4} [\d]{2}:[\d]{2}:[\d]{2}\.[\d]{6}[\s]+([\d]+)')
line = line.strip()
thread_id_match = regex_thread_id.search(line)
if thread_id_match:
self.thread_id = thread_id_match.group(1)
def checkKey(self, phase, key):
""" Check wheter the key is new and notify the listener in this case.
"""
if key == "lr":
key = "LearningRate"
key_registry = self.getKeys(phase)
if key not in key_registry:
key_registry.append(key)
for lis in self.listener:
lis.registerKey(phase, key)
def hasKey(self, phase, key):
""" Return true if the key was registered for the phase.
"""
return key in self.getKeys(phase)
Barista/backend/parser/parser_common.py 0000664 ڊ0011117 00000001617 13265130053 020472 0 ustar klemms o0cvpr
class ParserCommon(object):
# keys for train/test phase
TRAIN = 'TRAIN'
TEST = 'TEST'
def __init__(self):
super(ParserCommon, self).__init__()
self.listener = []
self.train_keys = []
self.test_keys = []
def addListener(self, listener):
"""Add the listener to the list of listener.
Every listener is notified about every parsing event.
"""
if listener not in self.listener:
self.listener.append(listener)
def removeListener(self, listener):
""" Remove the listener from the list of listener.
"""
self.listener.remove(listener)
def getKeys(self, phase):
""" Return all registered keys for the given phase.
"""
if phase == ParserCommon.TEST:
return self.test_keys
elif phase == ParserCommon.TRAIN:
return self.train_keys
Barista/backend/parser/parser_dummy.py 0000664 ڊ0011117 00000001347 13265130053 020335 0 ustar klemms o0cvpr from backend.parser.parser_common import ParserCommon
class ParserDummy(ParserCommon):
def __init__(self):
super(ParserDummy, self).__init__()
def sendParserFinished(self):
for lis in self.listener:
lis.parsingFinished()
def sendParserUpdate(self, phase, row):
for lis in self.listener:
lis.update(phase, row)
def sendParserHandle(self, event, line, groups):
for lis in self.listener:
lis.handle(event, line, groups)
def sendParserRegisterKeys(self, phase, key):
keyRegistry = self.getKeys(phase)
if key not in keyRegistry:
keyRegistry.append(key)
for lis in self.listener:
lis.registerKey(phase, key)
Barista/backend/parser/parser_listener.py 0000664 ڊ0011117 00000001031 13265130053 021015 0 ustar klemms o0cvpr
class ParserListener:
def update(self, phase, row):
""" Called when the parser has parsed a new record.
"""
pass
def handle(self, event, message, groups):
""" Called when the parser has parsed a registered event.
"""
pass
def registerKey(self, phase, key):
""" Called when a new key was found in the log data.
"""
pass
def parsingFinished(self):
""" Called when the parser has processed all available streams.
"""
pass
Barista/gui/caffe_version_manager/__init__.py 0000664 ڊ0011117 00000000000 13265130053 021552 0 ustar klemms o0cvpr Barista/gui/caffe_version_manager/caffe_version_manager.py 0000664 ڊ0011117 00000017317 13265130053 024341 0 ustar klemms o0cvpr from PyQt5.QtCore import pyqtSlot as Slot, QMetaObject,Qt
from PyQt5.QtGui import QIcon, QFont, QColor
from PyQt5.QtWidgets import (
QDialog,
QLabel,
QPushButton,
QListWidget,
QMessageBox,
QListWidgetItem,
QGridLayout,
QStyle,
qApp,
QAbstractItemView
)
from backend.barista.utils.logger import Log
from gui.caffepath_dialog import CaffepathDialog
import backend.barista.caffe_versions as caffeVersions
from gui.caffe_version_manager.caffe_version_widget import CaffeVersionWidget
from backend.networking.net_util import sendMsgToHost
from backend.networking.protocol import Protocol
from gui.gui_util import askFromList
class CaffeVersionManager(QDialog):
"""This Editor enables management of different caffe versions."""
def __init__(self, hostManager, default_actions, parent = None):
QDialog.__init__(self, parent)
self.project = None
self.setModal(True)
self.hostManager = hostManager
self.actions = default_actions
self.resize(600, 300)
self.layout = QGridLayout(self)
self.selectedVersion = None
self.boldFont = QFont()
self.boldFont.setBold(True)
"""Layout"""
self.pbAddVersion = QPushButton("Add new version")
self.layout.addWidget(self.pbAddVersion, 0, 0, 1, -1)
"""Version widgets"""
self.lstVersions = QListWidget(self)
self.lstVersions.setSelectionMode(QAbstractItemView.SingleSelection)
#self.lstVersions.setStyleSheet(self.lstVersions.styleSheet() + "QListWidget::item { border-bottom: 1px solid lightgray; }" )
self.layout.addWidget(self.lstVersions, 1, 0, -1, -1)
self.pbAddVersion.clicked.connect(self._onAddVersion)
self.lstVersions.itemSelectionChanged.connect(self._onSelectionChanged)
self.updateList()
def _onAddVersion(self):
"""Opens the dialog to add new caffe versions"""
hosts = self.hostManager.getActiveHostList()[:]
ret = None
if len(hosts) == 0:
caffedlg = CaffepathDialog("Add a new caffe version to Barista", "Add version")
caffedlg.exec_()
else:
hosts.insert(0, ["local", "Local host"])
ret = askFromList(self, hosts, "Select host on which to add version",
"You have configured remote Hosts.\nPlease select where to add the version.")
if ret:
if ret == "local":
caffedlg = CaffepathDialog("Add a new caffe version to Barista", "Add version")
caffedlg.exec_()
else:
host = self.hostManager.getHostById(ret)
caffedlg = CaffepathDialog("Add a new caffe version to the Host", "Add version", remote=True, host=host)
caffedlg.exec_()
self.updateList()
def _onRemoveVersion(self, name, host):
"""Removes the selected caffe version"""
if host == None:
if caffeVersions.getAvailableVersions() > 1:
caffeVersions.removeVersion(caffeVersions.getVersionByName(name))
if(self.project.getCaffeVersion() == name):
self.project.changeProjectCaffeVersion(caffeVersions.getDefaultVersion().getName())
else:
QMessageBox.warning(self, self.tr("Warning"), self.tr("Cannot delete version. At least one caffe-version must be available at all times in barista."))
else:
msg = {"key": Protocol.REMOVECAFFEVERSION, "versionname": name}
sendMsgToHost(host.host, host.port, msg)
self.updateList()
def _onSelectionChanged(self):
"""Displays more information about the selected caffe version if it is selected"""
if len(self.lstVersions.selectedItems()) >= 1 and self.lstVersions.currentItem() is not None:
widget = self.lstVersions.itemWidget(self.lstVersions.currentItem())
if not isinstance(widget, QLabel):
selected = {}
selected["host"] = widget.host
selected["versionname"] = widget.caffe_version.getName()
if self.versionsEqual(selected, self.selectedVersion):
self.selectedVersion = None
else:
self.selectedVersion = selected
self.updateList()
def versionsEqual(self, version1, version2):
try:
return version1["host"] == version2["host"] and version1["versionname"] == version2["versionname"]
except:
return False
def updateList(self):
"""updates the list containing all available caffe versions"""
self.lstVersions.clear()
lblLocalVersion = QLabel("Local versions:")
lblLocalVersion.setFont(self.boldFont)
self.addListWidget(lblLocalVersion, False, False)
localCurrent = ""
if self.project != None:
localCurrent = self.project.getCaffeVersion()
for version in caffeVersions.getAvailableVersions():
name = version.getName()
selected = self.versionsEqual(self.selectedVersion, {"host": None, "versionname": version.getName()})
widget = CaffeVersionWidget(caffeVersions.getVersionByName(name), self.lstVersions, self, isSelected = selected, current = name == localCurrent, restart = caffeVersions.restart, host = None)
self.addListWidget(widget, True, selected)
for host in self.hostManager.getActiveHostList():
host = self.hostManager.getHostById(host[0])
lblRemoteVersion = QLabel("Versions on " + host.host + ":"+ str(host.port) +":")
lblRemoteVersion.setFont(self.boldFont)
lblRemoteVersion.setStyleSheet(lblRemoteVersion.styleSheet() + "QLabel{ border-top: 1px solid black; }")
self.addListWidget(lblRemoteVersion, False, False)
remoteCurrent = ""
msg = {"key": Protocol.GETDEFAULTCAFFEVERSION}
reply = sendMsgToHost(host.host, host.port, msg)
if reply:
if reply["status"]:
remoteCurrent = reply["defaultVersionName"]
remoteRestart = False
msg = {"key": Protocol.GETCAFFERESTART}
reply = sendMsgToHost(host.host, host.port, msg)
if reply:
if reply["status"]:
remoteRestart = reply["cafferestart"]
msg = {"key": Protocol.GETCAFFEVERSIONS}
reply = sendMsgToHost(host.host, host.port, msg)
if reply:
remoteVersions = reply["versions"]
for version in remoteVersions:
version = caffeVersions.caffeVersion(version, remoteVersions[version]["root"], remoteVersions[version]["binary"], remoteVersions[version]["python"], remoteVersions[version]["proto"])
selected = self.versionsEqual(self.selectedVersion, {"host": host, "versionname": version.getName()})
widget = CaffeVersionWidget(version, self.lstVersions, self, isSelected = selected, current = remoteCurrent == version.getName(), restart = remoteRestart, host = host)
self.addListWidget(widget, True, selected)
def addListWidget(self, widget, selectable, selected):
item = QListWidgetItem()
item.setSizeHint(widget.sizeHint())
if not selectable:
item.setFlags(item.flags() & ~Qt.ItemIsSelectable)
if selected:
item.setBackground(QColor(233, 119, 72))
self.lstVersions.addItem(item)
self.lstVersions.setItemWidget(item, widget)
def updateProject(self, project):
"""Updates the caffe version of the project"""
self.project = project
self.updateList()
def showEvent(self, event):
self.updateList()
QDialog.showEvent(self, event)
Barista/gui/caffe_version_manager/caffe_version_widget.py 0000664 ڊ0011117 00000014201 13265130053 024177 0 ustar klemms o0cvpr from PyQt5.QtGui import QIcon
from PyQt5 import QtCore
from backend.barista.session.session import State
import backend.barista.caffe_versions as caffeVersions
from backend.networking.net_util import sendMsgToHost
from backend.networking.protocol import Protocol
from PyQt5.QtWidgets import (
QWidget,
QGridLayout,
QLabel,
QPushButton,
qApp,
QStyle,
QListWidget,
QListWidgetItem,
QAbstractItemView,
QMessageBox
)
class CaffeVersionWidget(QWidget):
"""This widget class represents a caffe version as it is displayed in the caffe_version_manager"""
def __init__(self, version, parent, versionManager, isSelected, current, restart, host):
super(CaffeVersionWidget, self).__init__(parent)
self.caffe_version = version
self.versionManager = versionManager
self.host = host
"""Icons"""
self.ico_delete = QIcon('resources/trash.png')
self.ico_edit = QIcon('resources/pencil')
self.ico_remove = QIcon('resources/remove.png')
"""Layout"""
self.layout = QGridLayout(self)
"""Name label"""
self.lblName = QLabel(self.caffe_version.getName())
self.layout.addWidget(self.lblName, 0, 0, 1, 1)
"""General buttons"""
self.btnDelete = QPushButton(self.ico_delete, "")
self.btnDelete.setToolTip("Delete Version")
self.btnDelete.setFixedSize(30,30)
self.layout.addWidget(self.btnDelete, 0, 1, 1, 1)
if current:
"""Indicators if version is current project version or not"""
if restart:
self.lblCurrent = QLabel("Restart!")
self.lblCurrent.setFixedSize(0, 0)
self.lblCurrent.setStyleSheet(self.lblCurrent.styleSheet() + "QLabel{background-color: #F4EE42; border: 1px solid black; border-radius: 5px}")
else:
self.lblCurrent = QLabel("Current")
self.lblCurrent.setFixedSize(0, 0)
self.lblCurrent.setStyleSheet(self.lblCurrent.styleSheet() + "QLabel{background-color: #00EE00; border: 1px solid black; border-radius: 5px}")
self.lblCurrent.setAlignment(QtCore.Qt.AlignCenter)
self.lblCurrent.setFixedSize(150, 30)
self.layout.addWidget(self.lblCurrent, 0, 2, 1, 1)
else:
self.btnSetCurrent= QPushButton("Set as current")
self.btnSetCurrent.setFixedSize(150, 30)
self.layout.addWidget(self.btnSetCurrent, 0, 2, 1, 1)
self.btnSetCurrent.clicked.connect(self._onSetCurrent)
if isSelected:
"""show details of the current version if it is selected"""
self.layout2 = QGridLayout()
#btnEdit = QPushButton(self.ico_edit, "")
#btnEdit.setFixedSize(30,30)
#self.layout2.addWidget(btnEdit, 0, 0, 1, 1)
self.layout2.addWidget(QLabel("Root:\t"+self.caffe_version.getRootpath()), 0, 1, 1, -1)
#btnEdit = QPushButton(self.ico_edit, "")
#btnEdit.setFixedSize(30,30)
#self.layout2.addWidget(btnEdit, 1, 0, 1, 1)
self.layout2.addWidget(QLabel("Binary:\t"+self.caffe_version.getBinarypath()), 1, 1, 1, -1)
#btnEdit = QPushButton(self.ico_edit, "")
#btnEdit.setFixedSize(30,30)
#self.layout2.addWidget(btnEdit, 2, 0, 1, 1)
self.layout2.addWidget(QLabel("Python:\t"+self.caffe_version.getPythonpath()), 2, 1, 1, -1)
#btnEdit = QPushButton(self.ico_edit, "")
#btnEdit.setFixedSize(30,30)
#self.layout2.addWidget(btnEdit, 3, 0, 1, 1)
self.layout2.addWidget(QLabel("Proto:\t"+self.caffe_version.getProtopath()), 3, 1, 1, -1)
self.layout.addLayout(self.layout2, 1, 0, 1, -1)
self.btnDelete.clicked.connect(self._onRemoveVersion)
def _onRemoveVersion(self):
"""Remove this version from the caffe_version_manager"""
self.versionManager._onRemoveVersion(self.caffe_version.getName(), self.host)
if self.host == None:
self.restartWarning()
else:
msgBox = QMessageBox(QMessageBox.Warning, "Warning", "Please restart Barista host for changes to apply, otherwise Barista may be unstable!")
msgBox.addButton("Ok", QMessageBox.NoRole)
msgBox.addButton("Restart now", QMessageBox.YesRole)
if msgBox.exec_() == 1:
msg = {"key": Protocol.RESTART, "pid": self.versionManager.project.getProjectId()}
sendMsgToHost(self.host.host, self.host.port, msg)
def _onSetCurrent(self):
"""Sets this version as the current projects/remote hosts caffe version"""
if self.host == None:
self.versionManager.project.changeProjectCaffeVersion(self.caffe_version.getName())
self.restartWarning()
caffeVersions.restart = True
else:
msg = {"key": Protocol.SETCURRENTCAFFEVERSION, "versionname": self.caffe_version.getName()}
sendMsgToHost(self.host.host, self.host.port, msg)
msgBox = QMessageBox(QMessageBox.Warning, "Warning", "Please restart Barista host for changes to apply, otherwise Barista may be unstable!")
msgBox.addButton("Ok", QMessageBox.NoRole)
msgBox.addButton("Restart now", QMessageBox.YesRole)
if msgBox.exec_() == 1:
msg = {"key": Protocol.RESTART, "pid": self.versionManager.project.getProjectId()}
ret = sendMsgToHost(self.host.host, self.host.port, msg)
if ret and not ret["status"]:
msgBox = QMessageBox(QMessageBox.Warning, "Warning", ret["error"][0])
msgBox.addButton("Ok", QMessageBox.NoRole)
msgBox.exec_()
self.versionManager.updateList()
def restartWarning(self):
msgBox = QMessageBox(QMessageBox.Warning, "Warning", "Please restart Barista client for changes to apply, otherwise Barista may be unstable!")
msgBox.addButton("Ok", QMessageBox.NoRole)
msgBox.addButton("Restart now", QMessageBox.YesRole)
if msgBox.exec_() == 1:
self.versionManager.actions.restart() Barista/gui/host_manager/__init__.py 0000664 ڊ0011117 00000000000 13265130053 017716 0 ustar klemms o0cvpr Barista/gui/host_manager/hardware_selector.py 0000664 ڊ0011117 00000013605 13265130053 021673 0 ustar klemms o0cvpr from PyQt5.QtWidgets import QDialog, QHBoxLayout,QVBoxLayout, QPushButton, QListWidget, QLabel, QListView, QScrollArea
from PyQt5.QtCore import QAbstractListModel, Qt
from PyQt5.QtGui import QFontDatabase
from backend.networking.net_util import sendMsgToHost, buildTransaction
from backend.networking.protocol import Protocol
class HardwareSelector(QDialog):
def __init__(self, host, port, parent = None):
QDialog.__init__(self, parent)
self.host = host
self.port = port
self._layout = QVBoxLayout(self)
self.setWindowTitle("Hardware on " + self.host + ":" + str(self.port))
self.setMinimumSize(800, 600)
self.hardware = None
self.selectedHW = None
self._model = None
self._lbl_status = QLabel()
self._layout.addWidget(self._lbl_status)
self._layoutH = QHBoxLayout()
self._layout.addLayout(self._layoutH)
self._hwlist = QListView()
self._layoutH.addWidget(self._hwlist)
self._lbl_hw = QLabel()
self._lbl_hw.setMinimumWidth(400)
self._lbl_hw.setMargin(10)
self._lbl_hw.setFont(QFontDatabase.systemFont(QFontDatabase.FixedFont))
#self._layoutH.addWidget(self._hwlabel)
self._hwscroll = QScrollArea()
self._hwscroll.setWidget(self._lbl_hw)
self._layoutH.addWidget(self._hwscroll)
self._layoutB = QHBoxLayout()
self._layout.addLayout(self._layoutB)
self._btn_select = QPushButton("Select")
self._layoutB.addWidget(self._btn_select)
self._btn_rescan = QPushButton("Rescan")
self._layoutB.addWidget(self._btn_rescan)
self._btn_cancel = QPushButton("Cancel")
self._layoutB.addWidget(self._btn_cancel)
self._btn_select.clicked.connect(self._select)
self._btn_cancel.clicked.connect(lambda: (self._quitScan(), self.close()))
self._btn_rescan.clicked.connect(self._prepareScan)
self._disable()
self._start()
def _start(self, ret=None):
if not ret:
self._prepareScan()
ret = self._getHardware()
if ret:
self.hardware = ret[0]
if len(self.hardware) == 0:
self._lbl_status.setText("No Hardware detected. Is Caffepath set?")
elif ret[1] == 0:
self._lbl_status.setText("Currently selected CPU: " + ret[0][0]["name"])
else:
self._lbl_status.setText("Currently selected GPU " + str(ret[1]) + ": " + ret[0][ret[1]]["name"])
self._model = self.HardwareListModel(self.hardware)
self._hwlist.setModel(self._model)
self._hwlist.selectionModel().currentChanged.connect(self._onSelection)
self._enable()
def _enable(self):
self._hwlist.setEnabled(True)
self._btn_rescan.setEnabled(True)
def _disable(self):
self._hwlist.setEnabled(False)
self._btn_select.setEnabled(False)
self._btn_rescan.setEnabled(False)
self._lbl_hw.setText("")
if self._hwlist.selectionModel():
self._hwlist.selectionModel().currentChanged.disconnect()
self._hwlist.setModel(None)
def _select(self):
self.selectedHW = self._hwlist.currentIndex().row()
self.close()
def _onSelection(self):
row = self._hwlist.currentIndex().row()
if row != -1:
self._lbl_hw.setText(self._getText(row))
self._lbl_hw.adjustSize()
self._btn_select.setEnabled(True)
else:
self._lbl_hw.setText("")
def _getText(self, row):
log = self.hardware[row]["log"]
text = ""
for l in log:
text += l + "\n"
return text
def _prepareScan(self):
self._disable()
ct = buildTransaction(self.host, self.port)
if ct:
self.ct = ct
msg = {"key": Protocol.SCANHARDWARE}
self.ct.send(msg)
self.ct.bufferReady.connect(self._processScan)
self.ct.socketClosed.connect(self._socketClosed)
self._lbl_status.setText("Start scanning...")
else:
self._lbl_status.setText("Failed to start Scan.")
def _processScan(self):
msg = self.ct.asyncRead()
if msg["status"]:
if "finished" in msg.keys():
if msg["finished"]:
self._quitScan()
self._lbl_status.setText("Scan finished.")
self._start([msg["hardware"], msg["current"]])
else:
if "id" in msg.keys():
self._lbl_status.setText("Found GPU " + str(msg["id"]) + ": " + msg["name"])
else:
self._lbl_status.setText("Found CPU: " + msg["name"])
else:
self._lbl_status.setText("Scan failed: " + msg["error"])
def _socketClosed(self):
self._quitScan()
self._enable()
self._lbl_status.setText("Connection lost!")
def _quitScan(self):
if hasattr(self, "ct"):
self.ct.bufferReady.disconnect()
self.ct.socketClosed.disconnect()
self.ct.close()
del self.ct
def _getHardware(self):
msg = {"key": Protocol.GETHARDWARE}
ret = sendMsgToHost(self.host, self.port, msg)
if ret and "hardware" in ret:
return [ret["hardware"], ret["current"]]
class HardwareListModel(QAbstractListModel):
def __init__(self, data):
QAbstractListModel.__init__(self)
self.data = data
def rowCount(self, parent=None, *args, **kwargs):
return len(self.data)
def data(self, index, role=None):
row = index.row()
if role == Qt.DisplayRole:
if row==0:
return "CPU: " + self.data[row]["name"]
return "GPU " + str(self.data[row]["id"]) + ": " + self.data[row]["name"]
Barista/gui/host_manager/host_manager.py 0000664 ڊ0011117 00000102611 13265130053 020641 0 ustar klemms o0cvpr import uuid
from PyQt5 import QtCore, QtWidgets
from PyQt5.QtCore import Qt, pyqtSlot as Slot, QMetaObject, QTimer, QObject
from PyQt5.QtGui import QIcon
from PyQt5.QtWidgets import *
from backend.barista.utils.settings import applicationQSetting
from backend.networking.net_util import sendMsgToHost, buildTransaction
from backend.networking.protocol import Protocol
from gui.gui_util import askFromList
from gui.host_manager.remote_file_dialog import RemoteFileDialog
from gui.host_manager.hardware_selector import HardwareSelector
from gui.manager_dialog import ManagerDialog
from backend.networking.barista_server import BaristaServer
class HostManager(ManagerDialog):
DEFAULT_HOSTNAME = 'Default'
def __init__(self, parent=None):
ManagerDialog.__init__(self, parent)
# main_window title
self.setWindowTitle("Host Manager")
# add "new host" button
self._pb_add = QPushButton("Add new Host")
self._buttonlayout.addWidget(self._pb_add)
self._buttonlayout.addStretch()
# combobox for filters
self._filterCombo = QtWidgets.QComboBox()
types = ["ALL", "ALIVE", "DEAD"]
self._filterCombo.addItems(types)
self._filterCombo.currentIndexChanged.connect(self._onFilter)
self._buttonlayout.addWidget(self._filterCombo)
# listwidget with all hosts
self._itemscroll.setSelectionMode(QAbstractItemView.ExtendedSelection)
self._itemscroll.setVerticalScrollMode(QAbstractItemView.ScrollPerPixel)
self._layout.addWidget(self._itemscroll)
self.dict = {"hostorder": [], "hostlist": {}}
self._pb_add.clicked.connect(self._addNewHost)
self._loadFromSettings()
self._addDefaultServer()
self._updateListWidget()
def _addDefaultServer(self):
""" Add a localhost server with name 'Default', if it's not already in the
list. """
for id, item in self.dict['hostlist'].iteritems():
if item.host == 'localhost' and item.port == BaristaServer.DEFAULT_PORT:
return False
host = Host(self, 'localhost', BaristaServer.DEFAULT_PORT, self.DEFAULT_HOSTNAME)
id = str(uuid.uuid4())
host.setID(id)
self.dict["hostorder"].append(id)
self.dict["hostlist"][id] = host
return True
def _loadFromSettings(self):
settings = applicationQSetting()
settings.beginGroup("Host")
val = settings.value("dict", "")
settings.endGroup()
if val != "":
self.dict = self._fromDict(val)
def _saveToSettings(self):
settings = applicationQSetting()
settings.beginGroup("Host")
settings.setValue("dict", self._toDict())
settings.endGroup()
def _toDict(self):
""" Converts the dict of Hosts into a serializable dictionary
"""
serialdict = {}
serialdict["hostlist"] = {}
for id in self.dict["hostorder"]:
host = self.dict["hostlist"][id]
serialdict["hostlist"][id] = {"host": host.host, "port": host.port, "name": host.name}
serialdict["hostorder"] = self.dict["hostorder"]
return serialdict
def _fromDict(self, dict):
""" Converts the serializable dictionary into the dict of Hosts
"""
hostdict = {}
hostdict["hostlist"] ={}
if "hostorder" in dict.keys():
hostdict["hostorder"] = dict["hostorder"]
for id in dict["hostorder"]:
host = Host(self, dict["hostlist"][id]["host"],
dict["hostlist"][id]["port"],
dict["hostlist"][id]["name"])
host.setID(id)
hostdict["hostlist"][id] = host
return hostdict
def _selectedHost(self):
if len(self._hostscroll.selectedItems()) < 1:
return None
id = self._hostscroll.selectedItems()[0].data(Qt.UserRole)
host = self.dict["hostlist"][id]
return host
def _selectionChange(self):
"""disable and enable the delete button on selection"""
num_selected = len(self._itemscroll.selectedItems())
self._pb_remove.setEnabled(num_selected > 0 and self._selectedHost().name != self.DEFAULT_HOSTNAME)
def getActiveHostList(self):
ret = []
for id in self.dict["hostorder"]:
if self.dict["hostlist"][id].connect():
host = [id, "Remote Host: '" + self.dict["hostlist"][id].name + "'"]
ret.append(host)
return ret
def getHostDataById(self, id):
if id in self.dict["hostorder"]:
return [self.dict["hostlist"][id].host, self.dict["hostlist"][id].port, self.dict["hostlist"][id].name]
def getHostById(self, id):
if id in self.dict["hostorder"]:
return self.dict["hostlist"][id]
def makePathRelative(self, id, path):
if id in self.dict["hostorder"]:
return self.makePathRealtiveDirect(self.dict["hostlist"][id].host,
self.dict["hostlist"][id].port, path)
def makePathRealtiveDirect(self, host, port, path):
msg = {"key": Protocol.MAKEPATHRELATIVE, "path": path}
ret = sendMsgToHost(host, port, msg)
if ret:
return ret["path"]
def _onFilter(self):
'''apply the filter to all items by hiding and unhiding'''
type = self._filterCombo.currentText()
activeHostIds = []
for host in self.getActiveHostList():
activeHostIds.append(host[0])
for index in range(0, self._itemscroll.count()):
item = self._itemscroll.item(index)
if (type == "DEAD" and item.data(Qt.UserRole) not in activeHostIds) \
or (type == "ALIVE" and item.data(Qt.UserRole) in activeHostIds) \
or type == "ALL":
item.setHidden(False)
else:
item.setHidden(True)
def onDelete(self):
"""remove selected hosts"""
ic = len(self._itemscroll.selectedItems())
if ic > 0:
# Prevent DEFAULT HOST from being deleted.
if self._selectedHost().name == self.DEFAULT_HOSTNAME:
return
gramNum = ""
if ic > 1:
gramNum = "s"
# confirm delete
ret = QMessageBox.question(self, "Remove {0} selected Host{1}".format(str(ic), gramNum),
"Do you really want to delete the selected host{}?".format(gramNum),
QMessageBox.Ok, QMessageBox.No)
if ret == QMessageBox.Ok:
for item in self._itemscroll.selectedItems():
self.deletebyID(item.data(Qt.UserRole))
# instead of using the _updateListWidget function it is more efficient to remove the widgets by hand
rows = []
# get all the rows
for item in self._itemscroll.selectedItems():
rows.append(self._itemscroll.row(item))
# remove in reverse order to conserve the row-number
for row in reversed(sorted(rows)):
item = self._itemscroll.takeItem(row)
item = None
self._saveToSettings()
self._updateSizeHint()
def onKeyDelete(self):
self.onDelete()
def keyReleaseEvent(self, QKeyEvent):
if QKeyEvent.key() == QtCore.Qt.Key_Delete:
self.onKeyDelete()
super(HostManager, self).keyReleaseEvent(QKeyEvent)
def deletebyID(self, id):
"""given an ID remove the entry from the host dict"""
if id:
if id in self.dict["hostorder"]:
self.dict["hostorder"].remove(id) # this removes it from the order list
self.dict["hostlist"][id].onDelete()
del self.dict["hostlist"][id]# this removes it from the dict
def removeFromListByID(self, id):
"""remove a HostWidget from the _itemscroll list"""
row = self._getIndexinList(id)
item = self._itemscroll.takeItem(row)
item = None
self._saveToSettings()
self._updateSizeHint()
def setName(self, id, name):
self.dict["hostlist"][id].name = name
self._saveToSettings()
self._updateSizeHint()
def _addNewHost(self):
mask = HostSelect(self)
mask.exec_()
ret = mask.returnvalue
if ret["valid"]:
host = Host(self, ret["host"], ret["port"], ret["host"])
id = str(uuid.uuid4())
host.setID(id)
self.dict["hostorder"].append(id)
self.dict["hostlist"][id] = host
self._saveToSettings()
self._updateListWidget()
def _updateListWidget(self):
self._itemscroll.clear()
for id in self.dict["hostorder"]:
host = self.dict["hostlist"][id]
widget = HostWidget(self)
widget.setHost(host)
host.widget = widget
widget.updateState()
item = QListWidgetItem()
item.setSizeHint(widget.sizeHint())
item.setData(Qt.UserRole, id)
item.setHidden(True)
self._itemscroll.addItem(item)
self._itemscroll.setItemWidget(item, widget)
self._updateSizeHint()
self._onFilter()
@Slot()
def _updateSizeHint(self):
"""update the size of a widget inside the listwidget"""
for index in range(0, self._itemscroll.count()):
item = self._itemscroll.item(index)
widget = self._itemscroll.itemWidget(item)
item.setSizeHint(widget.sizeHint())
self._itemscroll.update()
def createWidget(self, id):
# ... but you can't move the widget. you need to create the widget anew
host = self.dict["hostlist"][id]
widget = HostWidget(self)
host.widget = widget
widget.setHost(host)
widget.updateState()
return widget
def updateAfterMovement(self):
pass
def _getIndexinList(self, id):
"""given an id what is the index of the item in the listwidget"""
for index in range(0, self._itemscroll.count()):
item = self._itemscroll.item(index)
if item.data(Qt.UserRole) == id:
return index
return -1
def _showLineDialog(self, title, text, default=""):
"""display a simple dialog asking for a line input"""
# create a dialog
dialog = QDialog(self)
dialog.setWindowTitle(title)
# layouts and widgets
layout = QVBoxLayout(dialog)
label = QLabel(text)
layout.addWidget(label)
line = QLineEdit(default)
layout.addWidget(line)
button = QPushButton("Ok")
layout.addWidget(button)
button.clicked.connect(lambda: dialog.close())
# exec and get the line content
dialog.exec_()
return line.text()
class HostSelect(QDialog):
def __init__(self, parent):
self.hostlist = parent.dict["hostlist"]
QDialog.__init__(self, parent)
self.setWindowTitle("Add new Host")
self.layout = QVBoxLayout(self)
self.lbl_host = QLabel("Specifiy IP address or public hostname:")
self.layout.addWidget(self.lbl_host)
self.le_host = QLineEdit()
self.layout.addWidget(self.le_host)
self.lbl_port = QLabel("Specify port:")
self.layout.addWidget(self.lbl_port)
self.sb_port = QSpinBox()
self.sb_port.setMinimum(1)
self.sb_port.setSingleStep(1)
self.sb_port.setMaximum(65535)
self.sb_port.setButtonSymbols(QAbstractSpinBox.NoButtons)
self.layout.addWidget(self.sb_port)
self.sublayout = QHBoxLayout()
self.layout.addLayout(self.sublayout)
self.pb_confirm = QPushButton("OK")
self.sublayout.addWidget(self.pb_confirm)
self.pb_cancel = QPushButton("Cancel")
self.sublayout.addWidget(self.pb_cancel)
self.returnvalue = {"host": "", "port": 0, "valid": False}
self.pb_confirm.clicked.connect(self.checkInput)
self.pb_cancel.clicked.connect(self.close)
def checkInput(self):
if self.le_host.text() == "":
QMessageBox.critical(self, "Invalid", "Host can not be empty!")
return
for id in self.hostlist.keys():
if self.hostlist[id].host == self.le_host.text() and \
self.hostlist[id].port == self.sb_port.value():
QMessageBox.critical(self, "Already existing", "This host has already been added.")
return
self.returnvalue["host"] = self.le_host.text()
self.returnvalue["port"] = self.sb_port.value()
self.returnvalue["valid"] = True
self.close()
class HostWidget(QWidget):
def __init__(self, manager):
super(self.__class__, self).__init__()
self.manager = manager
self.host = None
# self.port = None
# self.name = ""
# self.id = None
# add layouts
self.layout = QVBoxLayout(self)
self.hbox0 = QHBoxLayout()
self.hbox0.setAlignment(Qt.AlignLeft)
self.hbox1 = QHBoxLayout()
self.hbox1.setAlignment(Qt.AlignLeft)
self.hbox2 = QHBoxLayout()
self.hbox2.setAlignment(Qt.AlignLeft)
self.hbox3 = QHBoxLayout()
self.hbox3.setAlignment(Qt.AlignLeft)
self.layout.addLayout(self.hbox0)
self.layout.addLayout(self.hbox1)
# add labels
self.lbl_status = QLabel("state")
self.lbl_status.setFixedWidth(45)
self.hbox0.addWidget(self.lbl_status)
# create all resources
ico_rename = QIcon("resources/pencil.png")
ico_remove = QIcon("resources/trash.png")
pix_reload = qApp.style().standardPixmap(QStyle.SP_BrowserReload)
ico_reload = QIcon(pix_reload)
pix_up = qApp.style().standardPixmap(QStyle.SP_ArrowUp)
ico_up = QIcon(pix_up)
pix_down = qApp.style().standardPixmap(QStyle.SP_ArrowDown)
ico_down = QIcon(pix_down)
#pix_path = qApp.style().standardPixmap(QStyle.SP_CommandLink)
#ico_path = QIcon(pix_path)
ico_path = QIcon("resources/gear.png")
pix_hw = qApp.style().standardPixmap(QStyle.SP_ComputerIcon)
ico_hw = QIcon(pix_hw)
pix_add = qApp.style().standardPixmap(QStyle.SP_FileIcon)
ico_add = QIcon(pix_add)
# make all buttons
self.pb_rename = QPushButton(ico_rename, "")
self.pb_rename.setFixedSize(30, 30)
self.pb_rename.setToolTip("Rename")
self.hbox1.addWidget(self.pb_rename)
# delete button
self.pb_remove = QPushButton(ico_remove, "")
self.pb_remove.setFixedSize(30, 30)
self.pb_remove.setToolTip("Remove")
self.hbox1.addWidget(self.pb_remove)
self.pb_reload = QPushButton(ico_reload, "")
self.pb_reload.setFixedSize(30, 30)
self.pb_reload.setToolTip("Reload")
self.hbox1.addWidget(self.pb_reload)
self.pb_up = QPushButton(ico_up, "")
self.pb_up.setFixedSize(30, 30)
self.pb_up.setToolTip("Move Up")
self.hbox1.addWidget(self.pb_up)
self.pb_down = QPushButton(ico_down, "")
self.pb_down.setFixedSize(30, 30)
self.pb_down.setToolTip("Move Down")
self.hbox1.addWidget(self.pb_down)
#self.pb_path = QPushButton(ico_path, "")
#self.pb_path.setFixedSize(30, 30)
#self.pb_path.setToolTip("Set Caffe Path")
#self.hbox1.addWidget(self.pb_path)
self.pb_hw = QPushButton(ico_hw, "")
self.pb_hw.setFixedSize(30, 30)
self.pb_hw.setToolTip("Manage server hardware")
self.hbox1.addWidget(self.pb_hw)
self.pb_add = QPushButton(ico_add, "")
self.pb_add.setFixedSize(30, 30)
self.pb_add.setToolTip("Add Session from Host")
self.hbox1.addWidget(self.pb_add)
# rest of labels
self.lbl_name = QLabel()
font = self.lbl_name.font()
font.setBold(True)
self.lbl_name.setFont(font)
self.hbox0.addWidget(self.lbl_name)
self.lbl_addr = QLabel()
font = self.lbl_addr.font()
font.setItalic(True)
self.lbl_addr.setFont(font)
self.hbox0.addWidget(self.lbl_addr)
self.lbl_hostconfig = QLabel()
self.hbox2.addWidget(self.lbl_hostconfig)
self.lbl_hosttraining = QLabel()
self.hbox2.addWidget(self.lbl_hosttraining)
self.layout.addLayout(self.hbox2)
self.lbl_hardwaremode = QLabel()
self.hbox3.addWidget(self.lbl_hardwaremode)
self.lbl_hardwarename = QLabel()
self.hbox3.addWidget(self.lbl_hardwarename)
self.layout.addLayout(self.hbox3)
#self.lbl_caffepath = QLabel()
#self.layout.addWidget(self.lbl_caffepath)
self.lbl_sessionpath = QLabel()
self.layout.addWidget(self.lbl_sessionpath)
self.lbl_sessioncount = QLabel()
self.layout.addWidget(self.lbl_sessioncount)
self.lbl_projectsession = QLabel()
self.layout.addWidget(self.lbl_projectsession)
self.lbl_projectsession_running = QLabel()
self.layout.addWidget(self.lbl_projectsession_running)
self.lbl_projectsession_finished = QLabel()
self.layout.addWidget(self.lbl_projectsession_finished)
self.lbl_projectsession_waiting = QLabel()
self.layout.addWidget(self.lbl_projectsession_waiting)
self.lbl_projectsession_other = QLabel()
self.layout.addWidget(self.lbl_projectsession_other)
self.lbl_connectioncount = QLabel()
self.layout.addWidget(self.lbl_connectioncount)
self.collapseGui()
# Number of sessions all together
# caffepath
# button: select caffe path
self.pb_rename.clicked.connect(self._rename, Qt.QueuedConnection)
self.pb_remove.clicked.connect(self._remove, Qt.QueuedConnection)
self.pb_reload.clicked.connect(self._onReload, Qt.QueuedConnection)
self.pb_up.clicked.connect(lambda: self.manager.moveUp("hostorder", self.host.id), Qt.QueuedConnection)
self.pb_down.clicked.connect(lambda: self.manager.moveDown("hostorder", self.host.id), Qt.QueuedConnection)
self.pb_hw.clicked.connect(self.showHardwareSelect)
self.pb_add.clicked.connect(self.addSessionFromHost)
# self.transaction = None
# self.timer = QTimer()
# self.timer.setSingleShot(True)
# self.timer.setInterval(15000)
# self.timer.timeout.connect(self._disconnected, Qt.QueuedConnection)
def showHardwareSelect(self):
self.hw = HardwareSelector(self.host.host, self.host.port, self)
self.hw.exec_()
if self.hw.selectedHW is not None:
ret = self.host.setHardware(self.hw.selectedHW)
if ret:
self.updateState()
if ret["status"]:
return
QMessageBox.critical(self, "Set Hardware", "Failed to set Hardware")
def addSessionFromHost(self):
if not hasattr(self.manager.parent.viewManager, "project"):
return
pid = self.manager.parent.viewManager.project.projectId
ret = self.host.getSessionsOfProject(pid)
seslist = None
if ret:
if ret["status"]:
if len(ret["sessions"]) > 0:
seslist = ret["sessions"]
else:
QMessageBox.information(self, "Sessions on '" + self.host.name + "'",
"No sessions for this project were found.")
else:
QMessageBox.critical(self, "Sessions on '" + self.host.name + "'",
"No connection could be established to this host!")
if seslist is None:
return
ses = askFromList(self, seslist, "Sessions on '" + self.host.name + "'",
"Please select a session to add.")
if ses is None:
return
self.manager.parent.viewManager.sessionController.loadRemoteSession((self.host.host, self.host.port, self.host.name), ses)
def setHost(self, host):
self.host = host
def _onReload(self):
self.manager.parent.viewManager.sessionController.filterState()
self.updateState()
def updateState(self):
# self._setState()
self.lbl_name.setText(self.host.name)
self.lbl_addr.setText("@ " + self.host.host + ":" + str(self.host.port))
self.host.updateNetwork()
def _rename(self):
'''ask for a new name an set it'''
name = self.manager._showLineDialog("Rename", "Enter a new name:", self.lbl_name.text())
self.lbl_name.setText(name)
self.manager.setName(self.host.id, name)
def _remove(self):
ret = QMessageBox.question(self.manager, "Remove this Host",
"Do you really want to delete this host?",
QMessageBox.Ok, QMessageBox.No)
if ret == QMessageBox.Ok:
self.manager.deletebyID(self.host.id)
self.manager.removeFromListByID(self.host.id)
def setState(self, alive=False):
if alive:
self.lbl_status.setText("alive")
else:
self.lbl_status.setText("dead")
# set the stylesheet
self.lbl_status.setStyleSheet("""
QLabel{
""" + self._statusColor(alive) + """
border-style: solid;
border-width: 1px;
border-color: #555555;
border-radius: 5px;
text-align: center;
}
""")
def _statusColor(self, status):
'''set the color for the status'''
if status:
return """background-color: #00EE00;color:#000000;"""
return """background-color:#EE0000;color:#EEEEEE;"""
def _setHardware(self, id, name):
color = """background-color: #00EECC;color:#000000;"""
if id is 0:
self.lbl_hardwaremode.setText("CPU")
color = """background-color: #FFEE00;color:#000000;"""
else:
self.lbl_hardwaremode.setText("GPU " + str(id-1))
self.lbl_hardwaremode.setStyleSheet("""
QLabel{
""" + color + """
border-style: solid;
border-width: 1px;
border-color: #555555;
border-radius: 5px;
text-align: center;
}
""")
self.lbl_hardwarename.setText(name)
def _setConfig(self, valid):
if valid:
self.lbl_hostconfig.setText("Valid config")
else:
self.lbl_hostconfig.setText("Invalid config")
self.lbl_hostconfig.setStyleSheet("""
QLabel{
""" + self._statusColor(valid) + """
border-style: solid;
border-width: 1px;
border-color: #555555;
border-radius: 5px;
text-align: center;
}
""")
def _setTraining(self, valid):
if valid:
self.lbl_hosttraining.setText("Host is free for training")
else:
self.lbl_hosttraining.setText("Training in progress")
self.lbl_hosttraining.setStyleSheet("""
QLabel{
""" + self._statusColor(valid) + """
border-style: solid;
border-width: 1px;
border-color: #555555;
border-radius: 5px;
text-align: center;
}
""")
def collapseGui(self):
obj = [self.lbl_sessionpath, self.lbl_hostconfig, self.lbl_hostconfig,
self.lbl_hosttraining, self.lbl_hardwaremode, self.lbl_hardwarename, self.lbl_sessioncount,
self.lbl_connectioncount]
obj = obj + [self.lbl_projectsession, self.lbl_projectsession_running, self.lbl_projectsession_waiting,
self.lbl_projectsession_finished, self.lbl_projectsession_other]
#obj = obj + [self.pb_path, self.pb_hw, self.pb_add]
for o in obj:
o.setHidden(True)
def _expandGui(self, projectsessions=False):
obj = [self.lbl_sessionpath, self.lbl_hostconfig, self.lbl_hostconfig,
self.lbl_hosttraining, self.lbl_hardwaremode, self.lbl_hardwarename, self.lbl_sessioncount,
self.lbl_connectioncount, self.lbl_projectsession]
if projectsessions:
obj = obj + [self.lbl_projectsession_running, self.lbl_projectsession_waiting,
self.lbl_projectsession_finished, self.lbl_projectsession_other]
#obj = obj + [self.pb_path, self.pb_hw, self.pb_add]
for o in obj:
o.setHidden(False)
# @Slot()
# def _updateNetwork(self):
# if self._connect():
# msg = {"key": Protocol.GETSTATUS}
# if hasattr(self.manager.parent, "viewManager"):
# if hasattr(self.manager.parent.viewManager, "project"):
# msg["projectid"] = self.manager.parent.viewManager.project.projectId
# self.transaction.send(msg)
# return
# self.collapseGui()
# self.setState()
# self.timer.stop()
# if self.host and self.port:
# msg = {"key": Protocol.GETSTATUS}
# if hasattr(self.manager.parent, "viewManager"):
# if hasattr(self.manager.parent.viewManager, "project"):
# msg["projectid"] = self.manager.parent.viewManager.project.projectId
#
#
# ret = sendMsgToHost(self.host, self.port, msg)
# if ret:
# if "data" in ret.keys():
# self._parseData(ret["data"])
#
# self.timer.start()
# return
#
def parseData(self, data):
self._expandGui("projectsessions" in data.keys())
self.setState(True)
if "sessionpath" in data.keys():
self.lbl_sessionpath.setText("Sessionpath: \t" + data["sessionpath"])
if "trainOnHW" in data.keys():
name = ""
if "hardware" in data.keys():
if len(data["hardware"]) >= data["trainOnHW"] and len(data["hardware"]) is not 0:
name = data["hardware"][data["trainOnHW"]]
self._setHardware(data["trainOnHW"], name)
if "sessioncount" in data.keys():
self.lbl_sessioncount.setText("Sessions on server: \t" + str(data["sessioncount"]))
if "connections" in data.keys():
self.lbl_connectioncount.setText("Connections open: \t" + str(data["connections"]))
if "config" in data.keys():
self._setConfig(data["config"])
if "training" in data.keys():
self._setTraining(data["training"])
if "projectsessions" in data.keys():
pdat = data["projectsessions"]
self.lbl_projectsession.setText("Sessions of this project: \t" + str(pdat["count"]))
self.lbl_projectsession_running.setText(" running/paused: \t" + str(pdat["running"]))
self.lbl_projectsession_finished.setText(" finished: \t\t" + str(pdat["finished"]))
self.lbl_projectsession_waiting.setText(" waiting: \t\t" + str(pdat["waiting"]))
other = pdat["count"] - pdat["finished"] - pdat["waiting"] - pdat["running"]
self.lbl_projectsession_other.setText(" other: \t\t" + str(other))
else:
self.lbl_projectsession.setText("Sessions of this project: \t0")
QMetaObject.invokeMethod(self.manager, "_updateSizeHint", Qt.QueuedConnection)
# def _connect(self):
# if self.transaction is None:
# ct = buildTransaction(self.host, self.port)
# if ct is not None:
# self.transaction = ct
# self.transaction.socketClosed.connect(self._disconnected)
# self.transaction.bufferReady.connect(self._heartbeat)
# self.timer.start()
# return True
# return self.transaction is not None
# def _disconnected(self):
# self.timer.stop()
# # self.transaction.socketClosed.disconnect()
# # self.transaction.bufferReady.disconnect()
# if self.transaction.isConnected():
# self.transaction.close()
# del self.transaction
# self.transaction = None
# self.collapseGui()
# self.setState()
# QMetaObject.invokeMethod(self.manager, "_updateSizeHint", Qt.QueuedConnection)
# def _heartbeat(self):
# ret = self.transaction.asyncRead()
# if ret is not None:
# if ret["key"] is Protocol.GETSTATUS:
# if "data" in ret.keys():
# self.timer.stop()
# self.timer.start()
# self.parseData(ret["data"])
#
# if hasattr(self.manager.parent, "viewManager"):
# if hasattr(self.manager.parent.viewManager, "project"):
# pid = self.manager.parent.viewManager.project.projectId
# if "projectsessions" in ret["data"].keys():
# if "pid" in ret["data"]["projectsessions"].keys():
# if not ret["data"]["projectsessions"]["pid"] == pid:
# self._updateNetwork()
# else:
# # broken protocol
# self._disconnected()
# else:
# self._updateNetwork()
class Host(QObject):
""" This class encapsulates a host known by Barista(GUI). All communication with
the actual server and corresponding data is managed here.
"""
def __init__(self, parent, host, port, name):
super(self.__class__, self).__init__()
self.manager = parent
self.widget = None #type: HostWidget
self.host = host
self.port = port
self.name = name
self.id = None
self.transaction = None
self.timer = QTimer()
self.timer.setSingleShot(True)
self.timer.setInterval(15000)
self.timer.timeout.connect(self.disconnected, Qt.QueuedConnection)
def setID(self, id):
self.id = id
def setHardware(self, hid):
msg = {"key": Protocol.SETHARDWARE, "hid": hid}
return sendMsgToHost(self.host, self.port, msg)
def getSessionsOfProject(self, pid):
msg = {"key": Protocol.GETSESSIONS, "pid": pid}
return sendMsgToHost(self.host, self.port, msg)
#def setCaffePath(self, path):
# msg = {"key": Protocol.SETCAFFEPATH, "path": path}
# return sendMsgToHost(self.host, self.port, msg)
def connect(self):
if self.transaction is None:
ct = buildTransaction(self.host, self.port)
if ct is not None:
self.transaction = ct
self.transaction.socketClosed.connect(self.disconnected)
self.transaction.bufferReady.connect(self.heartbeat)
self.timer.start()
return True
return self.transaction is not None
def disconnected(self):
self.timer.stop()
# self.transaction.socketClosed.disconnect()
# self.transaction.bufferReady.disconnect()
if self.transaction.isConnected():
self.transaction.close()
del self.transaction
self.transaction = None
self.widget.collapseGui()
self.widget.setState()
QMetaObject.invokeMethod(self.manager, "_updateSizeHint", Qt.QueuedConnection)
def heartbeat(self):
ret = self.transaction.asyncRead()
if ret is not None:
if ret["key"] is Protocol.GETSTATUS:
if "data" in ret.keys():
self.timer.stop()
self.timer.start()
self.widget.parseData(ret["data"])
if hasattr(self.manager.parent, "viewManager"):
if hasattr(self.manager.parent.viewManager, "project"):
pid = self.manager.parent.viewManager.project.projectId
if "projectsessions" in ret["data"].keys():
if "pid" in ret["data"]["projectsessions"].keys():
if not ret["data"]["projectsessions"]["pid"] == pid:
self.updateNetwork()
else:
# broken protocol
self.disconnected()
else:
self.updateNetwork()
@Slot()
def updateNetwork(self):
if self.connect():
msg = {"key": Protocol.GETSTATUS}
if hasattr(self.manager.parent, "viewManager"):
if hasattr(self.manager.parent.viewManager, "project"):
msg["projectid"] = self.manager.parent.viewManager.project.projectId
self.transaction.send(msg)
return
self.widget.collapseGui()
self.widget.setState()
def onDelete(self):
if self.transaction:
self.transaction.close()
self.transaction = None
Barista/gui/host_manager/remote_file_dialog.py 0000664 ڊ0011117 00000027152 13265130053 022011 0 ustar klemms o0cvpr from os.path import normpath
from PyQt5.QtCore import QAbstractTableModel, Qt, QModelIndex, pyqtSignal
from PyQt5.QtGui import QColor, QIcon
from PyQt5.QtWidgets import QDialog, QGridLayout, QTableView, QAbstractItemView, QComboBox, QPushButton, QLineEdit, \
QStyle, qApp, QFileIconProvider, QLabel
from backend.networking.net_util import sendMsgToHost
from backend.networking.protocol import Protocol
class RemoteFileDialog(QDialog):
def __init__(self, host, port, title="", fileFilter="All (*)", dirSelect=False, parent=None):
QDialog.__init__(self, parent)
self.fileFilter = fileFilter
# Set the window Title
if (title == ""):
title = "Remote"
windowtitle = title + " @ " + host + ":" + str(port)
self.setWindowTitle(windowtitle)
self.dirselect = dirSelect
self.returnvalue = ""
# Build the GUI
self.layout = QGridLayout(self)
self.tabview = self.KeyEventTableView()
self.tabview.verticalHeader().hide()
self.tabview.setSelectionMode(QAbstractItemView.SingleSelection)
self.tabview.setSelectionBehavior(QAbstractItemView.SelectRows)
self.tabview.setFocus()
self.layout.addWidget(self.tabview, 1, 0, 1, 4)
# Import Model
self.model = self.RemoteFileItemModel(host, port, self.dirselect)
self.tabview.setModel(self.model)
# Parent Button
pix_parentdir = qApp.style().standardPixmap(QStyle.SP_FileDialogToParent)
ico_parentdir = QIcon(pix_parentdir)
self.pb_parentdir = QPushButton(ico_parentdir, "")
self.pb_parentdir.setFixedSize(35, 35)
self.pb_parentdir.setFocusPolicy(Qt.NoFocus)
self.layout.addWidget(self.pb_parentdir, 0, 0)
# Refresh Button
pix_reload = qApp.style().standardPixmap(QStyle.SP_BrowserReload)
ico_reload = QIcon(pix_reload)
self.pb_refresh = QPushButton(ico_reload, "")
self.pb_refresh.setFixedSize(35, 35)
self.pb_refresh.setFocusPolicy(Qt.NoFocus)
self.layout.addWidget(self.pb_refresh, 0, 1)
# Current Dir
self.le_path = QLineEdit()
self.le_path.setEnabled(False)
self.le_path.setFocusPolicy(Qt.NoFocus)
self.layout.addWidget(self.le_path, 0, 2, 1, 2)
# File Filter
self.cb_filter = QComboBox()
self.cb_filter.addItems([fi.split()[0] for fi in fileFilter.split(";;")])
if len(fileFilter.split(";;")) == 1:
self.cb_filter.setEnabled(False)
self.layout.addWidget(self.cb_filter, 2, 3, 1, 1)
# Status Label
self.lbl_status = QLabel("")
self.layout.addWidget(self.lbl_status, 2, 0, 1, 3)
# Confirm Button
self.pb_select = QPushButton("Select")
self.pb_select.setFocusPolicy(Qt.NoFocus)
self.layout.addWidget(self.pb_select, 3, 2, 1, 1)
# Cancel Button
self.pb_cancel = QPushButton("Cancel")
self.pb_cancel.setFocusPolicy(Qt.NoFocus)
self.layout.addWidget(self.pb_cancel, 3, 3, 1, 1)
# Resize Tabel
self.tabview.setColumnWidth(0, 400)
self.tabview.hideColumn(1)
self.tabview.setColumnWidth(2, 100)
self.tabview.setColumnWidth(3, 200)
self.tabview.setMinimumWidth(715)
self.tabview.setMinimumHeight(400)
self.adjustSize()
# Connect Signals
self.tabview.doubleClicked.connect(self._processInput) # Confirm on doubleclick
self.tabview.enterKey.connect(self._processInput) # Confirm on enter/return
self.tabview.backKey.connect(self._goToParentDir) # ParentDir on Backspace
self.tabview.selectionModel().currentChanged.connect(self._updateLineEdit) # update current dir
self.model.updateStatus.connect(self._setStatus) # update Host connection status
self.cb_filter.currentIndexChanged.connect(self._updateFilter) # update FileFilter
self.pb_parentdir.clicked.connect(self._goToParentDir) # Button ParentDir
self.pb_refresh.clicked.connect(self._updateCurrentDir) # Button Refresh
self.pb_cancel.clicked.connect(self._cancel) # Button Cancel
self.pb_select.clicked.connect(self._processInputButton) # Button Confirm / Select
# Fill Model with /home
self.model.updateModel("/home/", self._createFilter())
self._updateLineEdit()
self._updateSelection()
def _processInput(self):
"""update path or accept selection on double click, enter/return key or button """
if (self._getCurrentItemType()):
self.model.updateModel(self._getCurrentItemPath(), self._createFilter())
self._updateLineEdit()
self.tabview.scrollToTop()
self._updateSelection()
else:
self._accept()
def _processInputButton(self):
if self.dirselect:
self.returnvalue = self.le_path.text()
self.close()
else:
self._processInput()
def _createFilter(self):
"""create the filter from the Qt-syntax based on the current selection"""
current_filter = self.fileFilter.split(";;")[self.cb_filter.currentIndex()]
current_filter = current_filter.split(" ", 1)[1]
current_filter = current_filter[1:len(current_filter) - 1]
current_filter = current_filter.split()
return current_filter
def _updateFilter(self):
"""update the model with the current filter"""
self.model.updateModel(self.model.currentPath, self._createFilter())
self._updateSelection()
def _updateLineEdit(self):
"""set the current dir in the line edit"""
self.le_path.setText(normpath(self.model.currentPath))
def _getCurrentItemType(self):
"""check if selection is dir"""
row = self.tabview.currentIndex().row()
index = self.model.createIndex(row, 1)
return self.model.data(index, Qt.UserRole)
def _getCurrentItemPath(self):
"""get the path of the current selection"""
row = self.tabview.currentIndex().row()
index = self.model.createIndex(row, 1)
item = self.model.itemData(index)
if len(item) == 0:
return ""
return item[Qt.DisplayRole]
def _updateSelection(self):
"""change the selection to the first item"""
self.tabview.scrollToTop()
self.tabview.setCurrentIndex(self.model.createIndex(0, 0, ))
def _goToParentDir(self):
"""go to the parent directory"""
path = self.model.currentPath + "/.."
self.model.updateModel(path, self._createFilter())
self._updateLineEdit()
self._updateSelection()
def _updateCurrentDir(self):
"""refresh current directory"""
self.model.updateModel(self.model.currentPath, self._createFilter())
self._updateLineEdit()
self._updateSelection()
def _cancel(self):
"""close this dialog"""
self.close()
def _accept(self):
"""close this dialog and write the returnvalue"""
self.returnvalue = self._getCurrentItemPath()
self.close()
def _setStatus(self, status):
"""set the host connection status in the status label"""
if status:
self.lbl_status.setText("")
else:
self.lbl_status.setText("Host unreachable!")
class KeyEventTableView(QTableView):
"""A small helper class to catch the Keys on the TableView"""
enterKey = pyqtSignal()
backKey = pyqtSignal()
def __init__(self):
QTableView.__init__(self)
def keyPressEvent(self, keyEvent):
"""Detect the key presses and fire signals"""
if keyEvent.key() == Qt.Key_Enter or keyEvent.key() == Qt.Key_Return:
self.enterKey.emit()
return
if keyEvent.key() == Qt.Key_Backspace:
self.backKey.emit()
return
super(self.__class__, self).keyPressEvent(keyEvent)
class RemoteFileItemModel(QAbstractTableModel):
"""Model to convert the file dictionary into a table"""
updateStatus = pyqtSignal(bool)
def __init__(self, host, port, dirselect):
QAbstractTableModel.__init__(self)
self.host = host
self.port = port
self.dirselect = dirselect
self.currentPath = ""
self.fileList = list()
def rowCount(self, QModelIndex_parent=None, *args, **kwargs):
"""return the current row count"""
return len(self.fileList)
def columnCount(self, QModelIndex_parent=None, *args, **kwargs):
"""reuturn the current column count"""
return 4
def data(self, index, role=None):
"""return the data of the table for different roles"""
row = index.row()
if (row < 0 or row >= len(self.fileList)):
return
col = index.column()
file = self.fileList[row]
if role == Qt.UserRole: # use the UserRole to check if item is dir
if file["isDir"]:
return True
return False
if role == Qt.DisplayRole: # the text that is shown
if col == 0:
return file["name"]
if col == 1:
return file["path"]
if col == 2:
return file["fileSize"]
if col == 3:
return file["lastChange"]
if role == Qt.DecorationRole: # the icon in the first column
if col == 0:
fip = QFileIconProvider()
if file["isDir"]:
ico = fip.icon(QFileIconProvider.Folder) # type: QIcon
return ico.pixmap(20, 20)
ico = fip.icon(QFileIconProvider.File) # type: QIcon
return ico.pixmap(20, 20)
if role == Qt.BackgroundColorRole: # make stripes
if row % 2 == 0:
return QColor(Qt.white)
return QColor(Qt.lightGray).lighter(120)
def headerData(self, p_int, orient, int_role=None):
"""Change the column names in the header"""
if int_role == Qt.DisplayRole and orient == Qt.Horizontal:
return ["Name", "Path", "FileSize", "Last Changed"][p_int]
def updateModel(self, path="/home/", fileFilter=["*"]):
"""update the model by connecting to the host and asking for a new dir dictionary"""
msg = {"key": Protocol.GETDIR, "path": path, "filter": fileFilter, "dirSelect": False}
if self.dirselect:
msg["dirSelect"] = True
res = sendMsgToHost(self.host, self.port, msg)
if res:
self.currentPath = res["path"]
oldlen = len(self.fileList)
newlen = len(res["data"])
diff = oldlen - newlen
if diff > 0:
self.beginRemoveRows(QModelIndex(), newlen, oldlen - 1)
self.fileList = res["data"] # has to be done here to prevent invalid index exception
self.endRemoveRows()
if diff < 0:
self.beginInsertRows(QModelIndex(), oldlen, newlen - 1)
self.fileList = res["data"] # has to be done here to prevent invalid index exception
self.endInsertRows()
if diff == 0:
self.fileList = res["data"]
self.dataChanged.emit(self.createIndex(0, 0), self.createIndex(newlen, 2))
self.updateStatus.emit(True)
return
self.updateStatus.emit(False)
Barista/gui/input_manager/__init__.py 0000664 ڊ0011117 00000000000 13265130053 020100 0 ustar klemms o0cvpr Barista/gui/input_manager/database_object.py 0000664 ڊ0011117 00000006550 13265130053 021453 0 ustar klemms o0cvpr from backend.input_db.lmdb_input import *
from backend.input_db.leveldb_input import *
from backend.input_db.hdf5_txt_input import *
class DatabaseObject:
'''abstraction to make accessing databases easy and consistent across types'''
def __init__(self):
self._legalTypes = ["LMDB", "LEVELDB", "HDF5TXT"]
self._db = None
self._path = None
self._dbType = None
self._openPath = None
self._openDBType = None
self._projectPath = None
def __del__(self):
'''on delete: close'''
if self._db is not None:
self._db.close()
def setDB(self, path, type):
'''set the path and type of a database.'''
if self.isLegalType(type):
self._path = unicode(path)
self._dbType = unicode(type)
def open(self):
'''open a database.'''
if self._db is not None:
self.close()
if self._path is not None and self.isLegalType(self._dbType):
if self._dbType == 'LMDB':
self._db = LmdbInput()
if self._dbType == 'LEVELDB':
self._db = LeveldbInput()
if self._dbType == 'HDF5TXT':
self._db = Hdf5TxtInput()
if self._projectPath:
self._db.setProjectPath(self._projectPath)
self._db.setPath(self._path)
self._db.open()
self._openPath = self._path
self._openDBType = self._dbType
def openFromPath(self, path, type):
'''directly open db from path.'''
if self.isLegalType(type):
self._path = unicode(path)
self._dbType = unicode(type)
self.open()
def close(self):
'''close the db.'''
if self._db is not None:
self._db.close()
self._db = None
self._openPath = None
self._openDBType = None
def getDataCount(self):
'''get the number of entries in the database.'''
if self._db is not None:
return self._db.getDataCount()
def getDimensions(self):
'''get the dimensions.'''
if self._db is not None:
return self._db.getDimensions()
def verifyConsistency(self):
'''check if all entries have the same channel number and size.
This may take a lot of time since every entry has to be checked.'''
if self._db is not None:
return self._db.verifyConsistency()
def getPath(self):
'''get the path of the current open db.
if no db is open get the path of the to be opened db.'''
if self._db is not None:
return self._openPath
return self._path
def getDBType(self):
'''get the type of the current open db.
if no db is open get the type of the to be opened db.'''
if self._db is not None:
return self._openDBType
return self._dbType
def isOpen(self):
'''check if a db is open.'''
if self._db is not None:
return self._db.isOpen()
return False
def isLegalType(self, type):
'''check if the passed type is legal.'''
if type in self._legalTypes:
return True
return False
def setProjectPath(self, path):
'''set the project path for the HDF5TXT format'''
if self._dbType == "HDF5TXT":
self._projectPath = path Barista/gui/input_manager/input_manager.py 0000664 ڊ0011117 00000217021 13265130053 021207 0 ustar klemms o0cvpr import os
import uuid
from PyQt5 import QtWidgets
from functools import partial
from PyQt5.QtCore import QUrl, QMetaObject, QDir, QSize
from PyQt5.QtCore import pyqtSlot as Slot
from PyQt5.QtGui import QDesktopServices
from gui.host_manager.remote_file_dialog import *
from gui.input_manager.database_object import *
from gui.manager_dialog import ManagerDialog
from gui.network_manager.layer_helper import *
from backend.barista.utils.logger import *
import backend.barista.utils.db_util as db_util
from backend.networking.net_util import sendMsgToHost
from backend.networking.protocol import Protocol
from gui.prototxt_editor.editor_widget import EditorWidget
from gui.gui_util import askFromList
from input_manager_dialogs import DatabaseCheckBoxDialog, AssignLayerCheckBoxDialog, TableEditor
from operator import itemgetter
from threading import Thread
from threading import Event
class InputManager(ManagerDialog):
# DB Dict
# databases = {id1 = {path = "..", "name="...", type=".."}, id2 =...}
# dborder = [id1,id2,id3]
def __init__(self, parent=None):
ManagerDialog.__init__(self, parent)
self.logid = Log.getCallerId("InputManager")
# main_window title
self.setWindowTitle("Input Manager")
# add db button
self._pb_add = QtWidgets.QPushButton("Add new Database")
self._buttonlayout.addWidget(self._pb_add)
# search and add button
self._pb_searchadd = QtWidgets.QPushButton("Search and add new Databases")
self._buttonlayout.addWidget(self._pb_searchadd)
# hdf5txt file button
self._pb_createtxt = QtWidgets.QPushButton("New HDF5TXT File")
self._buttonlayout.addWidget(self._pb_createtxt)
self._buttonlayout.addStretch()
# type label
self._type_label = QtWidgets.QLabel("Type:")
self._buttonlayout.addWidget(self._type_label)
# combobox for filters
self._filterCombo = QtWidgets.QComboBox()
types = ["ALL", "LMDB", "LEVELDB", "HDF5TXT"]
self._filterCombo.addItems(types)
self._filterCombo.currentIndexChanged.connect(self.onFilter)
self._buttonlayout.addWidget(self._filterCombo)
# listwidget with all dbs
self._itemscroll = self.ScrollList()
self._itemscroll.setSelectionMode(QtWidgets.QAbstractItemView.ExtendedSelection)
# layout settings
self._itemscroll.setVerticalScrollMode(QAbstractItemView.ScrollPerPixel)
self._layout.addWidget(self._itemscroll)
self._itemscroll.dropped.connect(self.fileDropEvent)
self.dbs = []
self.disable = False
self._pb_add.clicked.connect(self.opendb)
self._pb_searchadd.clicked.connect(self.searchAddDB)
self._pb_createtxt.clicked.connect(self.createTxt)
self.currentWalkDirectory = ""
# get all files with their type that lie under the top directory
self.fileList = []
def keyReleaseEvent(self, QKeyEvent):
if QKeyEvent.key() == QtCore.Qt.Key_Delete:
self.onKeyDelete()
super(InputManager, self).keyReleaseEvent(QKeyEvent)
"""
creates a new db entry for a selected database. this is handled differently depending on whether or not the db
source is local or remote by calling 'opendbLocal' or 'opendbRemote'
@author ..., j_stru18
"""
def opendb(self):
list = self.parent.hostManager.getActiveHostList() # type: list
if len(list) == 0:
self.opendbLocal()
else:
list.insert(0, ["local", "Local Host"])
ret = askFromList(self, list, "Select DB Source",
"You have configured remote Hosts.\nPlease select the Source of the Database.")
if ret:
if ret == "local":
self.opendbLocal()
else:
self.opendbRemote(ret)
"""
open a file dialog and create a new db entry with the resulting absolute path
if the selected file is *.hd5 or *.hdf5 a new *.txt file will be created, which contains the path of the selected
database file, because caffe expects a txt file
@author ..., f_prob02, j_stru18, l_hu0002
"""
def opendbLocal(self):
projectPath = self.parent.actions.getProjectPath()
if projectPath:
ret = QtWidgets.QFileDialog.getOpenFileName(self, "Select a Database", QDir.homePath(),
"ALL (*.mdb CURRENT *.ldb *.txt *.h5 *.hdf5);;" + "LMDB (*.mdb);;" +
"LEVELDB (CURRENT *.ldb);;" +
"HDF5 (*.h5 *.hdf5);;" +
"HDF5TXT (*.txt)")
# ret = [PATH, TYPE]
if len(ret[0]) != 0:
success = self.addDBbyPathLocal(ret[0], ret[1])
if success:
self.updateListWidget()
"""
does the same as opendbLocal but instead of the GUI File Dialog a RemoteFileDialog is used for the remote source
additionally the db dict that is added contains information about the host and port of the remote source
@author ..., f_prob02, j_stru18, l_hu0002
@param id the id of the remote source that was selected
"""
def opendbRemote(self, id):
data = self.parent.hostManager.getHostDataById(id)
if data:
filter = "ALL (*.mdb CURRENT *.ldb *.hdf5 *.h5 *.txt);;LMDB (*.mdb);;LEVELDB (CURRENT *.ldb);;" \
"HDF5 (*.h5 *.hdf5);;HDF5TXT (*.txt)"
rfd = RemoteFileDialog(data[0], data[1], "Select a Database", filter, parent=self)
rfd.exec_()
path = rfd.returnvalue
if rfd.returnvalue is not "":
if path and path != "":
success = self.addDBbyPathRemote(path, data[0], data[1])
if success: # a new db was added
self.updateListWidget()
def fileDropEvent(self, list):
"""called when an item is dragged and dropped onto the Input Manager List"""
for path in list:
if os.path.isdir(path):
self.searchAddDB(path)
elif os.path.isfile(path):
path, ext = os.path.splitext(path)
# make sure extension is valid, .ldb files need a CURRENT file
if not ext in [".mdb", ".txt", ""]:
continue
# make sure .mdb files are named data
if ext == ".mdb" and os.path.basename(path) != "data":
continue
# make sure if extension is empty the filename is CURRENT
if ext == "" and os.path.basename(path) != "CURRENT":
continue
# stick path and ext back together
path = path+ext
self.addDBbyPathLocal(path)
self.updateListWidget()
"""
Adds a db to the input manager's dict.
Parses for the file's type. In case of a .h5 or .hdf5 file a new HDF5TXT is created.
The db's files (in case of lmdb a single one and in case of leveldb and hdf5 multiple files) are hashed and used as
ID to prevent duplicates.
@author j_stru18
@param path : the path to the db file
@param typeString : if this method is called from openDBLocal, a typeString can be added as second parameter that
implies the dbs datatype
@return : whether or not the adding to the db dict was successful
"""
def addDBbyPathLocal(self, path, typeString=None):
filename = os.path.basename(path)
type = ""
# parse for type
if typeString is None or "ALL" in typeString:
type = db_util.getType(filename)
elif "LMDB" in typeString:
type = 'LMDB'
elif "LEVELDB" in typeString:
type = "LEVELDB"
elif "HDF5TXT" in typeString:
type = "HDF5TXT"
elif "HDF5" in typeString:
type = "HDF5"
newFileCreated = False
if type == "HDF5":
h5path = path
index = filename.rfind(".", 0, len(filename))
defaultName = filename[:index]
filename = self._showLineDialog("Create new HDF5TXT", "Enter a name for the HDF5TXT file:",
defaultName)
if filename is "": # Dialog was closed
return False
path = os.path.join(os.path.dirname(path), filename) + ".txt"
if os.path.exists(path):
Log.error("File already exists: " + path + ", can't create new HDF5TXT ", self.logid)
QtWidgets.QMessageBox.critical(self, "Can't create new HDF5TXT",
"File already exists:\n" + path)
return False
with open(path, 'w') as file:
# add a new line to be sure
file.write(h5path)
type = "HDF5TXT"
newFileCreated = True
# trim path for folder based dbs
if type == "LMDB" or type == "LEVELDB":
path = os.path.dirname(path) + os.path.sep
# create entry in db dict
db = dict()
db["name"] = type + " Database"
db["path"] = path
db["type"] = type
addingSuccessful = self._addDB(db)
if not addingSuccessful and newFileCreated: # adding the hdf5txt wasn't successful: remove newly created file
os.remove(path)
self.parent.networkManager.setModifiedFlag()
return addingSuccessful
"""
Adds a db to the input manager's dict.
Parses for the file's type. In case of a .h5 or .hdf5 file a new HDF5TXT is created.
@author j_stru18
@param path : the path to the db file
@param host : the hostname of the remote db file
@param port : the host's port
@return : whether or not the adding to the db dict was successful
"""
def addDBbyPathRemote(self, path, host, port):
filename = os.path.basename(path)
type = db_util.getType(filename)
hdf5ID = None
typeWasHDF5 = False
if type == "HDF5":
typeWasHDF5 = True
type = "HDF5TXT"
h5path = path
index = filename.rfind(".", 0, len(filename))
filename = filename[:index]
filename = self._showLineDialog("Create new HDF5TXT", "Enter a name for the HDF5TXT file:",
filename)
if filename is "": # Dialog was closed
return False
path = os.path.join(os.path.dirname(path), filename) + ".txt"
# we hash the hdf5 file: if it already exists in the input manager, we don't have to create a new HDF5TXT
msg = {"key": Protocol.GETHASH, "path": h5path, "type": "HDF5"}
ret = sendMsgToHost(host, port, msg)
if ret and ret["status"]:
hashValue = ret["hashValue"]
portHash = db_util.getStringHash(str(port))
hdf5ID = str(hashValue ^ portHash)
if not self.dbcontains(hdf5ID):
msg = {"key": Protocol.MAKEHDF5TXT, "path": path, "hdf5": h5path}
ret = sendMsgToHost(host, port, msg)
if not ret and not ret["status"]:
QtWidgets.QMessageBox.critical(self, "Can't create new HDF5TXT",
"Creation of File failed:\n"
+ path)
Log.error("Creation of HDF5TXT " + path + " failed", self.logid)
return False
else:
Log.error("Remote host sent no message.", self.logid)
if type == "LMDB" or type == "LEVELDB":
path = os.path.dirname(path) + os.path.sep
# create entry in db dict
db = dict()
db["name"] = type + " Database"
db["path"] = path
db["type"] = type
db["isRemote"] = True
db["host"] = host
db["port"] = port
if typeWasHDF5:
addingSuccessful = self._addDBbyID(db, hdf5ID)
else:
addingSuccessful = self._addDB(db)
self.parent.networkManager.setModifiedFlag()
return addingSuccessful
def searchAddDB(self, topdir=None):
""" searches a given Folder """
# if topdir is empty, the user has to select one, else topdir will be already given (e.g. drag and drop)
if not topdir:
# get Top directory and project path
topdir = QtWidgets.QFileDialog.getExistingDirectory(self, "Select a Database Folder", QDir.homePath())
if topdir == "": return # user canceled
#start database search in thread
e = Event()
cancel = Event()
thread = Thread(target=self.searchDB, args=(e, cancel, topdir, ))
#Messagebox contains information about current search and a cancel Button
cancelButton = QtWidgets.QPushButton("Cancel", self)
cancelButton.setText("Cancel")
cancelButton.clicked.connect(partial(self.walkCanceled, cancel))
msg = QtWidgets.QMessageBox(self)
msg.setText("Current Directory:")
msg.setInformativeText(self.currentWalkDirectory)
msg.setWindowTitle("Searching for databases...")
#add the cancel-Button with RejectRole = 1
msg.addButton(cancelButton, 1)
msg.setStyleSheet("QLabel{min-width: 700px; min-height: 50px}");
QtCore.QCoreApplication.processEvents()
msg.show()
thread.start()
#update the current os.walk directory in the messagebox
while(thread.is_alive()):
QtCore.QCoreApplication.processEvents()
e.wait()
msg.setInformativeText(self.currentWalkDirectory)
e.clear()
msg.close()
self.addDBtoInputManager(topdir)
#reset fileList to avoid multiple results of the same folders
self.fileList = []
def walkCanceled(self, cancel):
"""Set the Event to leave the current search for databases"""
cancel.set()
def searchDB(self, e, cancel, topdir=None):
"""Search for databases in given top directory. Execution as additional Thread"""
for root, dirs, files in os.walk(topdir):
for name in reversed(files):
#if the search is not canceled, search for databases with os.walk
if not cancel.isSet():
# parse for type
type = ""
# determine the type by analysing file Extensions
filePath, extension = os.path.splitext(os.path.join(root, name))
if extension == ".mdb":
type = "LMDB"
elif extension == ".txt":
type = "HDF5TXT"
elif extension == "":
if filePath[-7:] == "CURRENT":
type = "LEVELDB"
self.currentWalkDirectory = os.path.dirname(os.path.join(root, name))
e.set()
# if file has a valid type add it to the fileList
if len(type) > 0:
if type == "LMDB" and filePath[-4:] != "data":
pass # if db is from type lmdb but not named "data"
else:
self.fileList.append({"filepath": filePath+extension, "type": type})
#kill the current thread by leaving the function
else:
cancel.clear()
return
def addDBtoInputManager(self, topdir=None):
if not self.fileList:
QtWidgets.QMessageBox.warning(
self,
self.tr("Error: No database found"),
self.tr("Neither \"{}\" nor any subfolders contain valid Barista databases!".format(topdir))
)
return
# if there is just 1 item in the list (or the first item is a txt), skip the dialog
if len(self.fileList) > 1 or self.fileList[0]["type"] == "HDF5TXT":
# sort list by type
self.fileList = sorted(self.fileList, key=itemgetter("type"), reverse=True)
# call dialog
dialog = DatabaseCheckBoxDialog(self.fileList, topdir)
if dialog.exec_(): # checks if dialog was accepted ("ok pressed") or aborted ("cancel" pressed)
self.fileList = dialog.getFileList()
else:
return # if dialog was aborted, nothing needs to be imported
# add files to input manager
for file in self.fileList:
if file["type"] == "LMDB" or file["type"] == "LEVELDB":
file["filepath"] = os.path.dirname(file["filepath"]) + os.path.sep
elif file["type"] == "HDF5TXT":
file["filepath"] = file["filepath"]
db = dict()
db["name"] = file["type"] + " Database" if file["type"] != "HDF5TXT" else "HDF5 Database"
db["path"] = file["filepath"]
db["type"] = file["type"]
self._addDB(db)
self.updateListWidget()
self.parent.networkManager.setModifiedFlag()
"""
Create a new HDF5TXT file with the paths of one or more HDF5 files.
Opens a TableEditor where you can add, remove and edit lines and also select HDF5 files with a file dialog. This new
file can be saved, when you enter a name for it. When a file was created and its path exists, it is added to the
input manager, which will also be updated.
@author: j_schi48, j_stru18
"""
def createTxt(self):
self.editor = TableEditor()
newPath = self.editor.path
if newPath:
addingSuccessful = self.addDBbyPathLocal(self.editor.path) # create TXT is only implemented for local dbs
if addingSuccessful:
QtWidgets.QMessageBox.information(self, self.tr("File Saved"), self.tr("File saved at \"{}\"".format(newPath)))
self.updateListWidget()
def importdb(self, dict):
'''receive a new db dict, clear the old and open all'''
self.dict = None
self._initDict()
if len(dict) > 0:
k = dict.keys()
# for every entry
# check consistency
if "dborder" in k and "dblist" in k:
for dbid in dict["dborder"]:
if dbid in dict["dblist"]:
self._addDBbyID(dict["dblist"][dbid], dbid)
self.updateListWidget()
else:
Log.error("Invalid Data on Import", self.logid)
def deletebyID(self, id):
'''given an ID remove the entry from the db dict'''
if id:
if self.dbcontains(id):
self.dict["dborder"].remove(id) # this removes it from the order list
del self.dict["dblist"][id] # this removes it from the dict
self.parent.networkManager.setModifiedFlag() # notify change
assert not self.dbcontains(id)
def clearInputManager(self):
'''clear the inputmanager on changing the project'''
if self.dict is not None:
self.dict = None
self._initDict()
self.updateListWidget()
"""
add a new database to the dict. only for databases WITHOUT IDs
in case of a local db the relevant files are hashed and the hash is used as ID, in case of remote dbs a unique id
is generated
@author j_stru18
@param db a dictionary that holds the attributes of a database object
@return whether or not the adding of the db was successful
"""
def _addDB(self, db):
if not self._locationIsImported(db):
if "isRemote" in db and db["isRemote"]:
return self._addDBRemote(db)
else:
return self._addDBLocal(db)
else:
if "isRemote" in db and db["isRemote"]:
QtWidgets.QMessageBox.information(self, self.tr("Importing Failed"),
self.tr("There is already a file imported from \n\"{0}\"\n"
" at the host: \n{1}".format(db["path"], db["host"])))
else:
QtWidgets.QMessageBox.information(self, self.tr("Importing Failed"),
self.tr("There is already a file imported from \n\"{}\"".format(db["path"])))
return False
"""
add this new database to the dict. only for databases WITHOUT IDs
generates an ID of the database by hashing the file to ensure no database is added twice, unless it is a HDF5TXT
without any valid paths. In this case, a new unique ID is generated. This prevents unwanted hash conflicts for these
files.
@author ..., f_prob02, j_stru18, l_hu0002
@param db a dictionary that holds the attributes of a database object
@return True if the adding was correct, False if the same database was already loaded or if the type, name or path
was invalid
"""
def _addDBLocal(self, db):
isValidDB = False
if db["path"] is not None:
dbPaths = db_util.getDBPathsByType(db["path"], db["type"])
hashValue = db_util.getMultipleHash(dbPaths)
if hashValue == 0:
hashValue = uuid.uuid4()
id = str(hashValue)
isValidDB = self._addDBbyID(db, id)
return isValidDB
"""
Generates an ID by hashing a remote database and the remote host's port, which is used to add the database to the
input manager, unless it is a HDF5TXT without any valid paths. In this case, a new unique ID is generated and
modified with the hash of the port. This prevents hash unwanted conflicts for these files.
WARNING: until #401 is implemented, we have to hash the port of the remote host as well, to ensure that duplicate
files on different hosts can be added. This has to be changed afterwards.
@author: j_stru18
@param db a dictionary that holds the attributes of a remote database object
@return True if the adding was correct, False if the type, name or path was invalid
@pre ensure that db is a db from remote host
@pre ensure that db["path"] is absolute
"""
def _addDBRemote(self, db):
isValidDB = False
if db["path"] is not None:
msg = {"key": Protocol.GETHASH, "path": db["path"], "type": db["type"]}
ret = sendMsgToHost(db["host"], db["port"], msg)
if ret and ret["status"]:
hashValue = ret["hashValue"]
if hashValue == 0:
hashValue = uuid.uuid4().int
portHash = db_util.getStringHash(str(db["port"]))
id = str(hashValue ^ portHash)
isValidDB = self._addDBbyID(db, id)
else:
Log.error("Remote host sent no message.", self.logid)
return isValidDB
"""
Adds a new database to the input manager's dict for a given ID. When the DB is already loaded, the user is warned.
When the db dict isn't constructed in the right way and doesn't contain the right keys and doesn't have the right
type or is already imported, this method returns false. Else, if the adding was successful, it returns false.
@author : j_stru18
@param db : a dict containing the information of a database file
@id : the id of a database file as string
@return : whether or not the adding to the db dict was successful or not
"""
def _addDBbyID(self, db, id):
self._initDict()
addingSuccessful = False
if self._dbCheck(db):
if not self.dbcontains(id):
if "isRemote" not in db:
db["isRemote"] = False
self.dict["dborder"].append(id)
self.dict["dblist"].update({id: db})
addingSuccessful = True
else:
duplicatePath = self.getPathByID(id)
QtWidgets.QMessageBox.critical(self, "Duplicate Error on Import!", "The selected file {0} was already "
"imported:\n{1}".format(db["path"], duplicatePath))
Log.error("Duplicate error on import.", self.logid)
assert self.dbcontains(id)
return addingSuccessful
"""
returns the path of the saved db that is associated with the given ID
@author j_stru18
@param id a db's unique ID
@return the file path of the database that is associated with this ID
"""
def getPathByID(self, id):
path = None
self._initDict()
if id in self.dict["dborder"]:
path = self.dict["dblist"][id]["path"]
type = self.dict["dblist"][id]["type"]
if type == 'LMDB':
path = os.path.join(path, "data.mdb")
elif type == 'LEVELDB':
path = os.path.join(path, "CURRENT")
elif type == 'HDF5TXT': # in case of HDF5TXT the path already contains the filename
pass
return path
def dbcontains(self, id):
'''check if an ID is already in the dict'''
self._initDict()
if id in self.dict["dborder"]:
return True
return False
def _initDict(self):
'''this creates an empty dict if there is none'''
if self.dict is None:
self.dict = dict()
self.dict["dblist"] = dict()
self.dict["dborder"] = list()
"""
checks if a db dict is constructed in a valid way (contains keys for type, name and path and type is one of the
three expected types
@param db a dict containing the attributes of a database
"""
def _dbCheck(self, db):
validDB = True
if "type" not in db.keys():
Log.error("'type' not in Database key", self.logid)
validDB = False
elif "name" not in db.keys():
Log.error("'name' not in Database key", self.logid)
validDB = False
elif "path" not in db.keys():
Log.error("'path' not in Database key", self.logid)
validDB = False
elif db["type"] not in ["LMDB", "LEVELDB", "HDF5TXT"]:
Log.error("Database of wrong type: " + db["type"], self.logid)
validDB = False
return validDB
"""
Checks if there is already a db with the same location as the inout in the input managers dict of databases.
For local dbs the path is compared only; for remote dbs the host and port are taken into consideration as well.
@author: j_stru18
@param: dbToCheck: a dict containing the attributes of a database
@return: whether or not a database with the same location already exists in the input managers dict of databases
"""
def _locationIsImported(self, dbToCheck):
locationIsImported = False
for id, dict in self.dict["dblist"].iteritems():
if dict["path"] == dbToCheck["path"]:
if "isRemote" in dbToCheck and dbToCheck["isRemote"]:
if dict["isRemote"]:
# both are remote and share the same path and host/port
if (dict["host"] == dbToCheck["host"]) and (dict["port"] == dbToCheck["port"]):
locationIsImported = True
else:
if not dict["isRemote"]: # both are local and share the same path
locationIsImported = True
return locationIsImported
def getDBDict(self):
'''export the dict'''
self._initDict()
return self.dict
def getLogID(self):
'''return log id of input manager'''
return self.logid
def updateListWidget(self):
'''remove all the widgets from the listwidget and create new. easiest way of updating the list widget'''
self._itemscroll.clear()
# for every db in dict
for id in self.dict["dborder"]:
db = self.dict["dblist"][id]
# create a new widget and give it all necessary info
dbwidget = self.DatabaseWidget(db["type"], self)
dbwidget.setId(id)
dbwidget.setPath(db["path"])
dbwidget.setName(db["name"])
dbwidget.showUsage(self.parent.networkManager.network)
dbwidget.disableEditing(self.disable)
if db["isRemote"]:
dbwidget.setRemote(db["host"], db["port"])
dbwidget.open()
# create a new listwidget item and give it all necessary info
item = QtWidgets.QListWidgetItem(self._itemscroll)
item.setData(Qt.UserRole, id)
item.setData(255, db["type"])
item.setSizeHint(dbwidget.sizeHint())
# add the item and set the widget
self._itemscroll.addItem(item)
self._itemscroll.setItemWidget(item, dbwidget)
self.onFilter()
def onFilter(self):
'''apply the filter to all items by hiding and unhiding'''
type = self._filterCombo.currentText()
for index in range(0, self._itemscroll.count()):
item = self._itemscroll.item(index)
if item.data(255) == type or type == "ALL":
item.setHidden(False)
else:
item.setHidden(True)
def onButtonDelete(self, id = None):
'''delete db by id and update ListWidget'''
ret = QtWidgets.QMessageBox.question(self, "Remove selected Database",
"Do you really want to remove this database?",
QtWidgets.QMessageBox.Ok, QtWidgets.QMessageBox.No)
if ret == QtWidgets.QMessageBox.Ok:
if not id is None:
self.deletebyID(id)
self.updateListWidget()
def onKeyDelete(self):
selection = self._itemscroll.selectedItems()
count = len(selection)
if count > 0:
gramNum = ""
thisThese = "this"
if len(selection) > 1:
gramNum = "s"
thisThese = "these"
ret = QtWidgets.QMessageBox.question(self, "Remove {0} selected Database{1}".format(str(count), gramNum),
"Do you really want to remove {0} database{1}?".format(thisThese, gramNum),
QtWidgets.QMessageBox.Ok, QtWidgets.QMessageBox.No)
if ret == QtWidgets.QMessageBox.Ok:
for item in selection:
self.deletebyID(item.data(Qt.UserRole))
self.updateListWidget()
def setName(self, id, name):
'''change the name to a db'''
self.dict["dblist"][id]["name"] = name
self.parent.networkManager.setModifiedFlag()
self._updateSizeHint()
@Slot()
def _updateSizeHint(self):
'''update the size of a widget inside the listwidget'''
for index in range(0, self._itemscroll.count()):
item = self._itemscroll.item(index)
widget = self._itemscroll.itemWidget(item)
item.setSizeHint(widget.sizeHint())
self._itemscroll.update()
def _getIndexinList(self, id):
'''given an id what is the index of the item in the listwidget'''
for index in range(0, self._itemscroll.count()):
item = self._itemscroll.item(index)
if item.data(Qt.UserRole) == id:
return index
return -1
def createWidget(self, id):
# ... but you can't move the widget. you need to create the widget anew
dbwidget = self.DatabaseWidget(self.dict["dblist"][id]["type"], self)
dbwidget.setId(id)
dbwidget.setPath(self.dict["dblist"][id]["path"])
dbwidget.setName(self.dict["dblist"][id]["name"])
if self.dict["dblist"][id]["isRemote"]:
dbwidget.setRemote(self.dict["dblist"][id]["host"], self.dict["dblist"][id]["port"])
dbwidget.open()
return dbwidget
def updateAfterMovement(self):
self.parent.networkManager.setModifiedFlag()
# update list widget
self.updateListWidget()
def assignToLayer(self, id):
'''assign and unassign selected layers to the db given by id'''
# TODO: instead of setting the abs-path modify this in the future to allow:
# - use relative paths for the dbs in the dict
# - assign a reference to the layer to get the path at runtime from the input manager
# get type and path
type = self.dict["dblist"][id]["type"]
path = self.dict["dblist"][id]["path"]
# define corresponding layertypes
layertypes = []
if type == "LMDB" or type == "LEVELDB":
layertypes = ["Data"]
if type == "HDF5TXT":
layertypes = ["HDF5Data"]
# get the layers
network = self.parent.networkManager.network
mylayer = LayerHelper.getLayersInfo(layertypes, network, path)
if len(mylayer) == 0:
# no layer of this type
QtWidgets.QMessageBox.warning(self, type, "There is no layer of this type.")
return
# get the selected layers
dialog = AssignLayerCheckBoxDialog(mylayer)
if dialog.exec_():
sel = dialog.getSelectedLayersList()
else:
return
# set/remove the path
for layerid, layername, assigned in sel:
if assigned:
LayerHelper.setPathforType(layerid, path, type, network)
elif not [layerid, layername, assigned] in mylayer:
LayerHelper.setPathforType(layerid, "", type, network)
# update list widget
self.updateListWidget()
def _showLineDialog(self, title, text, default=""):
'''display a simple dialog asking for a line input'''
# create a dialog
dialog = QtWidgets.QDialog(self)
dialog.setWindowTitle(title)
# layouts and widgets
layout = QtWidgets.QVBoxLayout(dialog)
label = QtWidgets.QLabel(text)
layout.addWidget(label)
line = QtWidgets.QLineEdit(default)
layout.addWidget(line)
button = QtWidgets.QPushButton("Ok")
layout.addWidget(button)
button.clicked.connect(lambda: dialog.accept())
# exec and get the line content
isAccepted = dialog.exec_()
if isAccepted == QDialog.Accepted:
return line.text()
else:
return ""
def disableEditing(self, disable):
""" Set attribute disable """
self.disable = disable
class DatabaseWidget(QtWidgets.QWidget):
'''a widget to display a db in the listwidget'''
def __init__(self, type, manager, parent=None):
QtWidgets.QWidget.__init__(self, parent)
self.db = None
self.id = None
self.isRemote = False
self.remoteHost = None
# remember the manager
self.manager = manager
# add layouts
self.layout = QtWidgets.QVBoxLayout(self)
self.hbox1 = QtWidgets.QHBoxLayout()
self.hbox1.setAlignment(Qt.AlignLeft)
self.hbox2 = QtWidgets.QHBoxLayout()
self.hbox2.setAlignment(Qt.AlignLeft)
self.hbox3 = QtWidgets.QHBoxLayout()
self.hbox3.setAlignment(Qt.AlignLeft)
self.hbox4 = QtWidgets.QHBoxLayout()
self.hbox4.setAlignment(Qt.AlignLeft)
# add labels
self.lbl_remote = QtWidgets.QLabel()
self.lbl_path = QtWidgets.QLabel("path")
self.hbox3.addWidget(self.lbl_remote)
self.hbox3.addWidget(self.lbl_path)
self.layout.addLayout(self.hbox1)
self.layout.addLayout(self.hbox2)
self.layout.addLayout(self.hbox3)
self.layout.addLayout(self.hbox4)
# type label
self.lbl_type = QtWidgets.QLabel(type)
self.lbl_type.setObjectName("databaseType")
# Overwrite the type color by appending them to the current stylesheet.
self.lbl_type.setStyleSheet(
self.lbl_type.styleSheet() + " QLabel { background-color: "+self._typeColor(type)+" }")
self.lbl_type.setFixedWidth(100)
self.hbox1.addWidget(self.lbl_type)
# status label
self.lbl_status = QtWidgets.QLabel("stat")
self.lbl_status.setObjectName("databaseStatus")
self.lbl_status.setFixedWidth(45)
self.hbox1.addWidget(self.lbl_status)
self._updateStatus(False)
# name label
self.lbl_name = QtWidgets.QLabel("name")
font = self.lbl_name.font()
font.setBold(True)
self.lbl_name.setFont(font)
self.hbox1.addWidget(self.lbl_name)
# create all resources
ico_rename = QIcon("resources/pencil.png")
pix_loc = QtWidgets.qApp.style().standardPixmap(QtWidgets.QStyle.SP_DialogOpenButton)
ico_loc = QIcon(pix_loc)
pix_reload = QtWidgets.qApp.style().standardPixmap(QtWidgets.QStyle.SP_BrowserReload)
ico_reload = QIcon(pix_reload)
pix_up = QtWidgets.qApp.style().standardPixmap(QtWidgets.QStyle.SP_ArrowUp)
ico_up = QIcon(pix_up)
pix_down = QtWidgets.qApp.style().standardPixmap(QtWidgets.QStyle.SP_ArrowDown)
ico_down = QIcon(pix_down)
pix_assign = QtWidgets.qApp.style().standardPixmap(QtWidgets.QStyle.SP_CommandLink)
ico_assign = QIcon(pix_assign)
ico_remove = QIcon('resources/trash.png')
# make all buttons
self.pb_rename = QtWidgets.QPushButton(ico_rename, "")
self.pb_rename.setFixedSize(30, 30)
self.pb_rename.setToolTip("Rename")
self.hbox2.addWidget(self.pb_rename)
self.pb_remove = QtWidgets.QPushButton(ico_remove, "")
self.pb_remove.setFixedSize(30, 30)
self.pb_remove.setToolTip("Remove")
self.hbox2.addWidget(self.pb_remove)
self.pb_loc = QtWidgets.QPushButton(ico_loc, "")
self.pb_loc.setFixedSize(30, 30)
self.pb_loc.setToolTip("Change Location")
self.hbox2.addWidget(self.pb_loc)
self.pb_reload = QtWidgets.QPushButton(ico_reload, "")
self.pb_reload.setFixedSize(30, 30)
self.pb_reload.setToolTip("Reload")
self.hbox2.addWidget(self.pb_reload)
self.pb_up = QtWidgets.QPushButton(ico_up, "")
self.pb_up.setFixedSize(30, 30)
self.pb_up.setToolTip("Move Up")
self.hbox2.addWidget(self.pb_up)
self.pb_down = QtWidgets.QPushButton(ico_down, "")
self.pb_down.setFixedSize(30, 30)
self.pb_down.setToolTip("Move Down")
self.hbox2.addWidget(self.pb_down)
# some special buttons for HDF5TXT
if type == "HDF5TXT":
pix_edit = QtWidgets.qApp.style().standardPixmap(QtWidgets.QStyle.SP_FileDialogContentsView)
ico_edit = QIcon(pix_edit)
self.pb_edit = QtWidgets.QPushButton(ico_edit, "")
self.pb_edit.setFixedSize(30, 30)
self.pb_edit.setToolTip("Edit Textfile")
self.hbox2.addWidget(self.pb_edit)
# self.pb_open.clicked.connect(lambda: self._openFile(), Qt.QueuedConnection)
self.pb_edit.clicked.connect(lambda: (self.onEditPressed(), Qt.QueuedConnection))
pix_add = QtWidgets.qApp.style().standardPixmap(QtWidgets.QStyle.SP_FileDialogNewFolder)
ico_add = QIcon(pix_add)
self.pb_add = QtWidgets.QPushButton(ico_add, "")
self.pb_add.setFixedSize(30, 30)
self.pb_add.setToolTip("Add single Hdf5 File")
self.hbox2.addWidget(self.pb_add)
self.pb_add.clicked.connect(lambda: self._addHdf5(), Qt.QueuedConnection)
self.pb_assign = QtWidgets.QPushButton(ico_assign, "")
self.pb_assign.setFixedSize(30, 30)
self.pb_assign.setToolTip("Assign to Layer")
self.hbox2.addWidget(self.pb_assign)
# usage label
self.lbl_usage = QtWidgets.QLabel("no layer assigned")
font = self.lbl_usage.font()
font.setItalic(True)
self.lbl_usage.setFont(font)
self.hbox2.addWidget(self.lbl_usage)
# number of elements in the db
self.lbl_count = QtWidgets.QLabel("None")
self.lbl_count.setObjectName("databaseLabel")
self.hbox4.addWidget(self.lbl_count)
self.lbl_count.setMinimumWidth(200)
self.lbl_count.setSizePolicy(QtWidgets.QSizePolicy.Minimum, QtWidgets.QSizePolicy.Minimum)
self.lbl_count.setAlignment(Qt.AlignTop)
# dimensions of elements in the db
self.lbl_dim = QtWidgets.QLabel("None")
self.lbl_dim.setObjectName("databaseLabel")
self.hbox4.addWidget(self.lbl_dim)
self.lbl_dim.setMinimumWidth(200)
self.lbl_dim.setSizePolicy(QtWidgets.QSizePolicy.Minimum, QtWidgets.QSizePolicy.Minimum)
self.lbl_dim.setAlignment(Qt.AlignTop)
# connect the buttons
self.pb_rename.clicked.connect(self.onRename, Qt.QueuedConnection)
self.pb_loc.clicked.connect(self.changeLocation, Qt.QueuedConnection)
self.pb_reload.clicked.connect(self.reload, Qt.QueuedConnection)
self.pb_up.clicked.connect(lambda: self.manager.moveUp("dborder", self.id), Qt.QueuedConnection)
self.pb_down.clicked.connect(lambda: self.manager.moveDown("dborder", self.id), Qt.QueuedConnection)
self.pb_assign.clicked.connect(lambda: self.manager.assignToLayer(self.id), Qt.QueuedConnection)
self.pb_remove.clicked.connect(lambda: self.manager.onButtonDelete(self.id), Qt.QueuedConnection)
def getPath(self):
'''return the path of this db'''
return self.lbl_path.text()
def setPath(self, path):
'''set the path of this db'''
self.lbl_path.setText(path)
def getType(self):
'''get the type of this db'''
return self.lbl_type.text()
def setName(self, name):
'''set the name of this db'''
self.lbl_name.setText(name)
def setUsage(self, usage):
'''set the usage of this db'''
self.lbl_usage.setText(usage)
def setId(self, id):
'''set the id of this db'''
self.id = id
def setRemote(self, host, port):
'''set this as remote'''
self.isRemote = True
self.remoteHost = [host, port]
self.lbl_remote.setText(host + ":" + str(port))
def showUsage(self, network):
""" Display assignment of db to layers
"""
# define corresponding layertypes
layertypes = []
if self.getType() == "LMDB" or self.getType() == "LEVELDB":
layertypes = ["Data"]
if self.getType() == "HDF5TXT":
layertypes = ["HDF5Data"]
# get the layers
mylayer = LayerHelper.getLayersInfo(layertypes, network, self.getPath())
usedBy = ""
for _, layername, assigned in mylayer:
if assigned:
usedBy += layername + ", "
if usedBy is "":
self.setUsage("no layer assigned")
else:
self.setUsage(usedBy[:-2])
def changeLocation(self):
type = self.getType()
if type == "LMDB":
filter = "LMDB (*.mdb)"
elif type == "LEVELDB":
filter = "LEVELDB (CURRENT *.ldb)"
elif type == "HDF5TXT":
filter = "HDF5TXT (*.txt)"
else:
filter = "ALL"
path = QtWidgets.QFileDialog.getOpenFileName(self, "Select the new location", QDir.homePath(), filter)[0]
if len(path) == 0:
return
if type == "LMDB" or type == "LEVELDB":
path = os.path.dirname(path) + os.path.sep
self.manager.dict["dblist"][self.id]["path"] = path
self.dbo.setDB(path, type)
self.manager.updateListWidget()
"""
Reload this database widget by opening it again and in case of a HDF5TXT rehashing it. If the file was
changed outside of Barista, a warning is shown.
@author: j_stru18
"""
def reload(self):
self.open()
type = self.getType()
path = self.getPath()
if type == "HDF5TXT":
if not self.isRemote:
dbpaths = db_util.getHdf5List(path)
hashValue = db_util.getMultipleHash(dbpaths)
hashedID = str(hashValue)
else:
hostName = self.remoteHost[0]
port = self.remoteHost[1]
hashMsg = {"key": Protocol.GETHASH, "path": self.getPath(), "type": "HDF5TXT"}
hashRet = sendMsgToHost(hostName, port, hashMsg)
if hashRet["status"]:
hashValue = hashRet["hashValue"]
portHash = db_util.getStringHash(str(port))
if hashValue != 0: # file contained paths to HDF5 files
hashedID = str(hashValue ^ portHash)
else:
return
else:
Log.log("Reloading of database {} on remote host failed.".format(self.id),
self.manager.getLogID)
QtWidgets.QMessageBox.warning(self, "Reloading Failed", "Reloading of database on remote host"
" failed.")
return
if hashedID != self.id: # file was changed outside of Barista
QtWidgets.QMessageBox.warning(self, "File was Changed", "This HDF5TXT file's paths were changed "
"outside of Barista - check for duplicate "
"files may fail.")
def open(self):
'''try to open this db and fill all the label based on whether or not it is local or remote'''
if not self.isRemote:
self.openLocal()
else:
self.openRemote()
def openLocal(self):
'''try to open this db and fill all the labels with information'''
projectPath = self.manager.parent.actions.getProjectPath()
if projectPath:
# create a DatabaseObject and open it from path
self.dbo = DatabaseObject()
path = os.path.join(projectPath, self.getPath())
self.dbo.setDB(path, self.getType())
if self.getType() == "HDF5TXT":
self.dbo.setProjectPath(projectPath)
self.dbo.open()
# if DBO is open:
status = self.dbo.isOpen()
self._updateStatus(status)
if status:
# get the element count
count = self.dbo.getDataCount()
self.lbl_count.setText(self._convertDatacount(count))
# get the element dim
dim = self.dbo.getDimensions()
self.lbl_dim.setText(self._convertDimensions(dim))
else:
self.lbl_count.setText("None")
self.lbl_dim.setText("None")
# close the DBO
self.dbo.close()
# call the manager to update the size hints
# use this threadsave methode to enable updates of the GUI
QMetaObject.invokeMethod(self.manager, "_updateSizeHint", Qt.QueuedConnection)
else:
Log.error("No project set, can't build rel. paths", self.manager.getLogID())
def openRemote(self):
''' try to open this db on a remote host and fill all the label with information'''
msg = {"key": Protocol.GETDBSTATUS, "type": self.getType(), "path": self.getPath()}
ret = sendMsgToHost(self.remoteHost[0], self.remoteHost[1], msg)
if ret and ret["status"]: # getting db status was successful
self._updateStatus(True)
# get the element count
count = ret["dataCount"]
self.lbl_count.setText(self._convertDatacount(count))
# get the element dim
dim = ret["dimensions"]
self.lbl_dim.setText(self._convertDimensions(dim))
else: # getting db status was not successful
self._updateStatus(False)
self.lbl_count.setText("None")
self.lbl_dim.setText("None")
QMetaObject.invokeMethod(self.manager, "_updateSizeHint", Qt.QueuedConnection)
"""
Opens a view of the current HDF5TXT that can be edited.
In case of a local database, a TableEditor is shown where all lines are marked as either dead or alive paths and
where these lines can be added, removed or altered.
In case of a remote database, the text file is simply opened in a file dialog and can be changed and saved.
@TODO: implement the behavior with the TableEditor for remote files as well (Task #2 of #449)
@author: j_schi48, j_stru18
@ensure: the type is HDF5TXT
"""
def onEditPressed(self):
if not self.isRemote:
self._editLocal()
else:
self._editRemote()
self.open()
"""
Method for editing local HDF5TXT files.
The user can edit the file using a TableEditor. If the file was changed and is no duplicate of a file, that is
already imported, the entry with the old hashed ID is deleted and it is added with the new ID. If changes were
made and the modified file is a duplicate, then the user has the choice of either discarding all changes or
replacing the old duplicate with the modified file.
@author: j_stru18
@ensure: the database of this DatabaseWidget is remote
"""
def _editLocal(self):
path = self.getPath()
allLines = db_util.getLinesAsList(path)
h5paths = db_util.getHdf5List(path)
if len(h5paths) != 0:
oldHash = db_util.getMultipleHash(h5paths)
oldID = str(oldHash)
else:
oldID = self.id
self.editor = TableEditor(self.getPath()) # actual editing is done here
newh5paths = db_util.getHdf5List(path)
if len(newh5paths) != 0: # modified file is not empty
newHash = db_util.getMultipleHash(newh5paths)
newID = str(newHash)
else:
if len(h5paths) != 0: # file wasn't empty, but it is now
newID = str(uuid.uuid4())
else:
newID = self.id
isFileDuplicate = self.manager.dbcontains(newID) # does a db with the same id exist?
if newID != oldID:
if isFileDuplicate: # new file is identical to another imported file
duplicatePath = self.manager.getPathByID(newID)
ret = QtWidgets.QMessageBox.question(self, "HDF5TXT with same Files Found",
"Another file with the same ID was found: \n{}\n Do you want to keep the changed "
"HDF5TXT and remove the old duplicate from Barista?".format(duplicatePath),
QtWidgets.QMessageBox.Apply, QtWidgets.QMessageBox.Discard)
if ret == QtWidgets.QMessageBox.Apply: # remove both old databases and add modified new one
self.manager.deletebyID(newID)
self.manager.deletebyID(oldID)
self.manager.addDBbyPathLocal(path)
assert self.manager.dbcontains(newID)
Log.log("Old HDF5TXT was removed and current file was modified.",
self.manager.getLogID())
self.manager.updateListWidget()
else: # discard changes, old status is restored
with open(path, 'w') as file:
for line in allLines:
file.write(line)
Log.log("No changes to HDF5TXT files were made.", self.manager.getLogID())
else: # file changed, but is unique
self.manager.deletebyID(oldID)
self.manager.addDBbyPathLocal(path)
"""
Method for editing files on remote hosts.
The user can edit the file using a file dialog. If the file was changed and is no duplicate of a file, that is
already imported, the entry with the old hashed ID is deleted and it is added with the new ID. If changes were
made and the modified file is a duplicate, then the user has the choice of either discarding all changes or
replacing the old duplicate with the modified file.
WARNING: until #401 is implemented, we have to hash the port of the remote host as well, to ensure that
duplicate files on different hosts can be added. This has to be changed afterwards.
TODO: when #449 was fully implemented with a TableEditor for remote files, this method has to be updated.
@author: j_stru18
@ensure: the database of this DatabaseWidget is remote
"""
def _editRemote(self):
hostName = self.remoteHost[0]
port = self.remoteHost[1]
path = self.getPath()
hashMsg = {"key": Protocol.GETHASH, "path": path, "type": "HDF5TXT"}
oldHashRet = sendMsgToHost(hostName, port, hashMsg)
fileContentMsg = {"key": Protocol.GETFILECONTENT, "path": path}
fileContent = sendMsgToHost(hostName, port, fileContentMsg)
if not oldHashRet["status"]:
Log.error("Hashing of old HDF5TXT not successful.", self.manager.getLogID())
return
if not fileContent["status"]:
Log.error("Getting file content from host was not successful.", self.manager.getLogID())
return
oldHashValue = oldHashRet["hashValue"]
portHash = db_util.getStringHash(str(port))
if oldHashValue == 0: # file contained no paths to HDF5 files
oldID = self.id
else:
oldID = str(oldHashValue ^ portHash)
self._openFileRemote() # actual file editing
newHashRet = sendMsgToHost(hostName, port, hashMsg) # hash file again for new ID
newHashValue = newHashRet["hashValue"]
if newHashValue != 0: # modified file is not empty
newID = str(newHashValue ^ portHash)
else:
if oldHashValue != 0: # file wasn't empty, but is now
newID = str(uuid.uuid4())
else:
newID = self.id
isFileDuplicate = self.manager.dbcontains(newID) # does a db with the same id exist?
if newID != oldID:
if isFileDuplicate: # new file is identical to another imported file
duplicatePath = self.manager.getPathByID(newID)
ret = QtWidgets.QMessageBox.question(self, "HDF5TXT with same Files Found",
"Another file with the same ID was found: \n{}\n Do you want to keep the changed "
"HDF5TXT and remove the old duplicate from Barista?".format(duplicatePath),
QtWidgets.QMessageBox.Apply, QtWidgets.QMessageBox.Discard)
if ret == QtWidgets.QMessageBox.Apply: # remove both old databases and add modified new one
self.manager.deletebyID(newID)
self.manager.deletebyID(oldID)
self.manager.addDBbyPathRemote(path, hostName, port)
assert self.manager.dbcontains(newID)
Log.log("Old HDF5TXT was removed and current file was modified.",
self.manager.getLogID())
self.manager.updateListWidget()
else: # discard changes, old status is restored
self._editorSave(fileContent["file"])
Log.log("No changes to HDF5TXT files were made.", self.manager.getLogID())
else: # file changed, but is unique
self.manager.deletebyID(oldID)
self.manager.addDBbyPathRemote(path, hostName, port)
def onRename(self):
'''ask for a new name and set it'''
name = self.manager._showLineDialog("Rename", "Enter a new name:", self.lbl_name.text())
self.setName(name)
self.manager.setName(self.id, name)
def _convertDatacount(self, count):
strcount = "Number of Elements:"
if count:
for key in sorted(count.keys()):
strcount += "\n" + key + ":"
strcount += "\n\t" + str(count[key])
else:
strcount += "\nNone"
return strcount
def _convertDimensions(self, dim):
strdim = "Dimensions:"
if dim:
for key in sorted(dim.keys()):
strdim += "\n" + key + ":"
l = dim[key]
if len(l) == 0:
strdim += "\n\tscalar"
else:
# reformat
strdim += "\n" + str(len(l)) + ":\t"
for i in range(0, len(l)):
strdim += str(l[i])
if i < len(l) - 1:
strdim += " x "
else:
strdim += "\nNone"
return strdim
def _updateStatus(self, status):
'''update the status label'''
if status:
self.lbl_status.setText("alive")
else:
self.lbl_status.setText("dead")
# Overwrite the status colors by appending them to the current stylesheet.
self.lbl_status.setStyleSheet(
self.lbl_status.styleSheet() + "QLabel {"+self._statusColor(status)+"}"
)
def _typeColor(self, type):
'''define the colors for every type as strings'''
if type == "LMDB":
return "#FFBF00"
if type == "LEVELDB":
return "#FF8000"
if type == "HDF5TXT":
return "#2EFEF7"
return "#eeeeee"
def _statusColor(self, status):
'''set the color for the status'''
if status:
return """background-color: #00EE00;color:#000000;"""
return """background-color:#EE0000;color:#EEEEEE;"""
def _openFile(self):
if self.isRemote:
self._openFileRemote()
else:
self._openFileLocal()
def _openFileLocal(self):
'''open the file of HDF5TXT in a system prototxt_editor'''
if self.getType() == "HDF5TXT":
projectPath = self.manager.parent.actions.getProjectPath()
if projectPath:
path = os.path.join(projectPath, self.getPath())
if os.path.exists(path):
url = QUrl(path, QUrl.TolerantMode)
QDesktopServices.openUrl(url)
else:
Log.error("File not found: " + path, self.manager.getLogID())
QtWidgets.QMessageBox.critical(self, "File not Found!", "File not Found at:\n" + path)
self.open()
def _openFileRemote(self):
msg = {"key": Protocol.GETFILECONTENT, "path": self.getPath()}
ret = sendMsgToHost(self.remoteHost[0], self.remoteHost[1], msg)
if ret["status"]:
self.editor = EditorWidget(False, self.manager)
self.editor.setText(ret["file"])
self.editor.setWindowTitle(self.getPath())
self.editor.sgSave.connect(self._editorSave)
self.editor.sgClose.connect(self._editorClose)
self.editor.exec_()
else:
QtWidgets.QMessageBox.critical(self, "Open File", "Failed to open remote file:\n" + self.getPath())
def _editorClose(self):
self.editor.sgSave.disconnect(self._editorSave)
self.editor.sgClose.disconnect(self._editorClose)
self.editor.deleteLater()
self.editor = None
def _editorSave(self, text):
msg = {"key": Protocol.WRITEFILECONTENT, "path": self.getPath(), "file": text}
ret = sendMsgToHost(self.remoteHost[0], self.remoteHost[1], msg)
if ret["status"]:
return
QtWidgets.QMessageBox.critical(self, "Write File", "Failed to write remote file:\n" + self.getPath())
"""
adds a new .h5 or .hdf5 file to an existing HDF5TXT file
in case of a local db, the file or files are hashed and it is checked if the resulting HDF5TXT already exists
TODO: remove this method, when #449 was fully implemented with a TableEditor for remote files
@author j_stru18
"""
def _addHdf5(self):
if self.isRemote:
self._addHdf5Remote()
else:
self._addHdf5Local()
"""
Adds a new HDF5 File to the local HDF5TXT file:
After selecting one or more files , it checks whether or not there is another imported db that contains paths
to the same file as the selected HDF5TXT with the newly selected paths would. If the modified file would be no
duplicate, it is deleted under its old ID and added again with its new one.
If it is a duplicate, the user is asked whether to keep the changes and remove the old file or discard them and
modify none of them. When changes were made to either one of the files, the Widgets of the input manager are
reloaded.
TODO: remove this method, when #449 is fully implemented with a TableEditor for remote files
@author j_stru18
"""
def _addHdf5Local(self):
if self.getType() != "HDF5TXT": # guard statement for wrong db type
Log.error("No HDF5TXT file was selected.", self.manager.getLogID())
return
ret = QtWidgets.QFileDialog.getOpenFileNames(self, "Select a HDF5 Database", QDir.homePath(),
"HDF5 (*.hdf5 *.h5)")
path = self.getPath()
newh5paths = ret[0]
if len(newh5paths) == 0: # guard statement for empty selection
Log.error("File not found: " + path, self.manager.getLogID())
QtWidgets.QMessageBox.critical(self, "File not Found!", "File not Found at:\n" + path)
return
if os.path.exists(path): # there are selected files and the HDF5TXT exists
h5paths = db_util.getHdf5List(path)
newHash = db_util.getMultipleHash(newh5paths)
if not h5paths: # file contains no valid paths
oldID = self.id
else:
oldHash = db_util.getMultipleHash(h5paths)
newHash += oldHash
oldID = str(oldHash)
newID = str(newHash)
isFileDuplicate = self.manager.dbcontains(newID) # does a db with the same id exist?
updateNeeded = False
if not isFileDuplicate:
with open(path, 'a+') as file:
for newh5path in newh5paths: # add paths to HDF5TXT
file.write("\n" + newh5path)
self.manager.deletebyID(oldID)
self.manager.addDBbyPathLocal(path)
assert self.manager.dbcontains(newID)
updateNeeded = True # db entries were changed and the inputs need to be updated
else:
duplicatePath = self.manager.getPathByID(newID)
ret = QtWidgets.QMessageBox.question(self, "HDF5TXT with same Files Found",
"Another file with the same ID was found: \n{}\n Do you want to keep the changed "
"HDF5TXT and remove the old duplicate from Barista?".format(duplicatePath),
QtWidgets.QMessageBox.Apply, QtWidgets.QMessageBox.Discard)
if ret == QtWidgets.QMessageBox.Apply: # remove both old databases and add modified new one
with open(path, 'a+') as file:
for newh5path in newh5paths: # add paths to HDF5TXT
file.write("\n" + newh5path)
self.manager.deletebyID(newID)
self.manager.deletebyID(oldID)
self.manager.addDBbyPathLocal(path)
assert self.manager.dbcontains(newID)
updateNeeded = True # db entries were changed and the inputs need to be updated
Log.log("Old HDF5TXT was removed and current file was modified.",
self.manager.getLogID())
else: # discard changes
Log.log("No changes to HDF5TXT files were made.", self.manager.getLogID())
if updateNeeded:
self.manager.updateListWidget()
"""
Adds a new HDF5 File to the local HDF5TXT file:
After selecting a .h5 or .hdf5 file , it checks whether or not there is another imported db that contains paths
to the same file as the selected HDF5TXT with the newly selected path would. If the modified file would be no
duplicate, it is deleted under its old ID and added again with its new one.
If it is a duplicate, the user is asked whether to keep the changes and remove the old file or discard them and
modify none of them. When changes were made to either one of the files, the Widgets of the input manager are
reloaded.
WARNING: until #401 is implemented, we have to hash the port of the remote host as well, to ensure that
duplicate files on different hosts can be added. This has to be changed afterwards.
TODO: remove this method, when #449 was fully implemented with a TableEditor for remote files
@author j_stru18
"""
def _addHdf5Remote(self):
hostName = self.remoteHost[0]
port = self.remoteHost[1]
path = self.getPath()
rfd = RemoteFileDialog(hostName, port, "Select a HDF5 Database", "HDF5 (*.hdf5 *.h5)")
rfd.exec_()
newPath = rfd.returnvalue
if newPath == "": # guard statement for empty path
Log.error("No HDF5TXT file was selected.", self.manager.getLogID())
return
oldHashMsg = {"key": Protocol.GETHASH, "path": path, "type": "HDF5TXT"}
oldHashRet = sendMsgToHost(hostName, port, oldHashMsg)
newHashMsg = {"key": Protocol.GETHASH, "path": newPath, "type": "HDF5"}
newHashRet = sendMsgToHost(hostName, port, newHashMsg)
if not oldHashRet["status"]: # guard statement for invalid return status #1
Log.error("Hashing of old HDF5TXT not successful.", self.manager.getLogID())
return
if not newHashRet["status"]: # guard statement for invalid return status #2
Log.error("Hashing of selected HDF5 was not successful", self.manager.getLogID())
return
oldHashValue = oldHashRet["hashValue"]
portHash = db_util.getStringHash(str(port))
if oldHashValue == 0: # file contained no paths to HDF5 files
oldID = self.id
else:
oldID = str(oldHashValue ^ portHash)
updateNeeded = False
newHashValue = oldHashValue + newHashRet["hashValue"]
newID = str(newHashValue ^ portHash)
isFileDuplicate = self.manager.dbcontains(newID) # does a db with the same id exist?
if not isFileDuplicate:
addPathMsg = {"key": Protocol.ADDHDF5, "path": path, "hdf5": newPath}
addPathRet = sendMsgToHost(hostName, port, addPathMsg)
if addPathRet["status"]:
self.manager.deletebyID(oldID)
self.manager.addDBbyPathRemote(path, hostName, port)
assert self.manager.dbcontains(newID)
updateNeeded = True # db entries were changed and the inputs need to be updated
else:
Log.error("Failed to add HDF5 Path: " + newPath, self.manager.getLogID())
else: # file with identical hashed ID exists
duplicatePath = self.manager.getPathByID(newID)
ret = QtWidgets.QMessageBox.question(self, "HDF5TXT with same Files Found",
"Another file with the same ID was found: \n{}\n Do you want to keep the changed "
"HDF5TXT and remove the old duplicate from Barista?".format(duplicatePath),
QtWidgets.QMessageBox.Apply, QtWidgets.QMessageBox.Discard)
if ret == QtWidgets.QMessageBox.Apply: # remove both old databases and add modified new one
addPathMsg = {"key": Protocol.ADDHDF5, "path": path, "hdf5": newPath}
addPathRet = sendMsgToHost(hostName, port, addPathMsg)
if addPathRet["status"]:
self.manager.deletebyID(oldID)
self.manager.deletebyID(newID)
self.manager.addDBbyPathRemote(path, hostName, port)
assert self.manager.dbcontains(newID)
updateNeeded = True # db entries were changed and the inputs need to be updated
Log.log("Old HDF5TXT was removed and current file was modified.",
self.manager.getLogID())
else:
Log.error("Failed to add HDF5 Path: " + newPath, self.manager.getLogID())
else: # discard changes
Log.log("No changes to HDF5TXT files were made.", self.manager.getLogID())
if updateNeeded:
self.manager.updateListWidget()
def disableEditing(self, disable):
""" Disable the button to assign a new layer """
self.pb_assign.setDisabled(disable)
# class to make the Database List accept Drag and Drop inputs
class ScrollList(QtWidgets.QListWidget):
dropped = pyqtSignal(list)
def __init__(self, parent=None):
QtWidgets.QWidget.__init__(self, parent)
self.setAcceptDrops(True)
def dragEnterEvent(self, event):
if event.mimeData().hasUrls:
event.accept()
else:
event.ignore()
def dragMoveEvent(self, event):
if event.mimeData().hasUrls:
event.accept()
else:
event.ignore()
def dropEvent(self, event):
if event.mimeData().hasUrls:
event.setDropAction(QtCore.Qt.CopyAction)
event.accept()
links = []
for url in event.mimeData().urls():
links.append(str(url.toLocalFile()))
self.dropped.emit(links)
else:
event.ignore()
Barista/gui/input_manager/input_manager_dialogs.py 0000664 ڊ0011117 00000041036 13265130053 022712 0 ustar klemms o0cvpr import copy
import os
from PyQt5 import QtWidgets
from PyQt5 import QtGui
from PyQt5 import QtCore
from gui.host_manager.remote_file_dialog import *
class SelectableListView(QtWidgets.QListView):
"""This ListView is able to toggle and untoggle the current item, which row was clicked anywhere"""
clickedSignal = QtCore.pyqtSignal()
def __init__(self, parent):
self.parent = parent
QtWidgets.QListView.__init__(self, parent)
self.clickedSignal.connect(self.toggleClicked)
def toggleClicked(self):
#toggle the checkbox for the selected row
selectedRows = self.selectedIndexes()
if len(selectedRows) == 1:
selectedItem = self.parent.listModel.item(selectedRows[0].row())
checkState = Qt.Checked if selectedItem.checkState() == Qt.Unchecked else Qt.Unchecked
item = self.parent.listModel.item(selectedRows[0].row())
item.setCheckState(checkState)
def mouseReleaseEvent(self, event):
#if a single click appears, the user have to toggle the checkbox itself
super(SelectableListView, self).mouseReleaseEvent(event)
def mouseDoubleClickEvent(self, event):
#with a doubleclick the user can toggle the checkbox by clicking anywhere in the row
self.clickedSignal.emit()
class CheckBoxDialog(QtWidgets.QDialog):
""" A Dialog presenting items to the user as a List with Checkboxes
"""
def __init__(self, parent=None):
QtWidgets.QDialog.__init__(self, parent)
self.setWindowModality(Qt.ApplicationModal)
# layouts
self.layout = QtWidgets.QVBoxLayout(self)
self.buttonLayout = QtWidgets.QHBoxLayout()
# elements
self.label = QtWidgets.QLabel("")
self.list = SelectableListView(self)
self.list.setSelectionMode(QtWidgets.QListView.ExtendedSelection)
self.listModel = QtGui.QStandardItemModel(self)
self.list.setModel(self.listModel)
self.pb_ok = QtWidgets.QPushButton("Ok")
self.pb_can = QtWidgets.QPushButton("Cancel")
self.pb_toggle = QtWidgets.QPushButton("Toggle Selected")
self.pb_selall = QtWidgets.QPushButton("Check All")
self.pb_selnone = QtWidgets.QPushButton("Check None")
# add elements to layouts
self.buttonLayout.addWidget(self.pb_selall)
self.buttonLayout.addWidget(self.pb_selnone)
self.buttonLayout.addWidget(self.pb_toggle)
self.buttonLayout.addStretch(40)
self.buttonLayout.addWidget(self.pb_ok)
self.buttonLayout.addWidget(self.pb_can)
self.layout.addWidget(self.label)
self.layout.addWidget(self.list)
self.layout.addLayout(self.buttonLayout)
# connect the buttons
self.pb_selall.clicked.connect(self.selectAll)
self.pb_selnone.clicked.connect(self.selectNone)
self.pb_toggle.clicked.connect(self.toggleSelected)
self.pb_ok.clicked.connect(self.accept)
self.pb_can.clicked.connect(self.close)
self.pb_ok.setDefault(True)
self.pb_ok.setFocus()
def selectAll(self):
for index in range(self.listModel.rowCount()):
item = self.listModel.item(index)
item.setCheckState(Qt.Checked)
def selectNone(self):
for index in range(self.listModel.rowCount()):
item = self.listModel.item(index)
item.setCheckState(Qt.Unchecked)
def toggleSelected(self):
selectedRows = self.list.selectedIndexes()
if len(selectedRows) > 0:
firstItem = self.listModel.item(selectedRows[0].row())
checkState = Qt.Checked if firstItem.checkState() == Qt.Unchecked else Qt.Unchecked
for index in selectedRows:
item = self.listModel.item(index.row())
item.setCheckState(checkState)
class DatabaseCheckBoxDialog(CheckBoxDialog):
""" A Dialog presenting found files to the user as a List with Checkboxes
can return a List with only selected Items in it"""
def __init__(self, fileList, root, parent=None):
CheckBoxDialog.__init__(self, parent)
self.setWindowTitle("Choose Databases")
self.resize(700, 500)
self.label.setText("Choose the Databases you want to add")
self.fileList = fileList
self.root = root
# add content to the list
if self.fileList:
if isinstance(self.fileList[0], type(str())):
self.fillStr()
elif isinstance(self.fileList[0], type(dict())):
self.fillAddSearch()
def fillAddSearch(self):
for file in self.fileList:
path = os.path.relpath(file["filepath"], self.root+"/..")
item = QtGui.QStandardItem(path)
item.setToolTip(file["filepath"])
item.setCheckable(True)
item.setCheckState(Qt.Checked)
item.setEditable(False)
self.listModel.appendRow(item)
def fillStr(self):
for file in self.fileList:
item = QtGui.QStandardItem(file)
item.setCheckable(True)
item.setCheckState(Qt.Checked)
item.setEditable(False)
self.listModel.appendRow(item)
def getFileList(self):
# delete all unchecked items from the list
for index in reversed(range(self.listModel.rowCount())):
item = self.listModel.item(index)
if item.isCheckable() and item.checkState() == QtCore.Qt.Unchecked:
del self.fileList[index]
return self.fileList
class AssignLayerCheckBoxDialog(CheckBoxDialog):
""" A Dialog presenting layers to the user as a List with Checkboxes
can return a list of layer's id, name and their check state"""
def __init__(self, itemsList, parent=None):
CheckBoxDialog.__init__(self, parent)
self.setWindowTitle("Choose Layers")
self.resize(400, 300)
self.label.setText("Assign layers to the database")
self.itemsList = copy.deepcopy(itemsList)
# add content to the list
if self.itemsList:
self.createCheckboxes()
def createCheckboxes(self):
for layerid, layername, assigned in self.itemsList:
item = QtGui.QStandardItem(layername)
item.setCheckable(True)
if assigned:
item.setCheckState(Qt.Checked)
else:
item.setCheckState(Qt.Unchecked)
item.setEditable(False)
self.listModel.appendRow(item)
def getSelectedLayersList(self):
# delete all unchecked items from the list
for index in reversed(range(self.listModel.rowCount())):
item = self.listModel.item(index)
if item.isCheckable():
self.itemsList[index][2] = (item.checkState() == Qt.Checked)
return self.itemsList
class TableEditor(QtWidgets.QDialog):
def __init__(self, path=False, parent=None):
QtWidgets.QDialog.__init__(self, parent)
# get labels whether there is a given path or not
if path:
windowTitle = "HDF5 Textfile Editor" + " - " + os.path.basename(path)
savelabel = "Save"
else:
windowTitle = "HDF5 Textfile Editor"
savelabel = "Save as"
# window settings
self.setWindowTitle(windowTitle)
self.setWindowModality(Qt.ApplicationModal)
self.resize(500, 600)
# layouts
self.layout = QtWidgets.QVBoxLayout(self)
self.topLayout = QtWidgets.QHBoxLayout()
self.buttonLayout = QtWidgets.QHBoxLayout()
# buttons
self.label = QtWidgets.QLabel("Add, remove and edit lines below:")
self.pb_save = QtWidgets.QPushButton(savelabel)
self.pb_cancel = QtWidgets.QPushButton("Cancel")
self.pb_addline = QtWidgets.QPushButton("Add Line")
self.pb_rmline = QtWidgets.QPushButton("Remove Line")
self.pb_addfile = QtWidgets.QPushButton("Add File(s)")
# table
self.table = self.CustomTable()
self.table.insertColumn(0)
self.table.insertColumn(1)
self.table.setHorizontalHeaderLabels(["Path", "Availibility"])
self.icon_exists = QtGui.QIcon("resources/valid.png")
self.icon_missing = QtGui.QIcon("resources/invalid.png")
header = self.table.horizontalHeader()
header.setSectionResizeMode(0, QtWidgets.QHeaderView.Stretch)
header.setSectionResizeMode(1, QtWidgets.QHeaderView.ResizeToContents)
# textfile
self.path = path
if path:
try:
self.txtfile = open(self.path, "r").read().splitlines()
for counter, line in enumerate([line for line in self.txtfile if line]):
self.table.insertRow(counter)
self.table.setItem(counter, 0, QtWidgets.QTableWidgetItem(line))
self.fillCheckItem(counter, line)
except IOError:
return
# add elements to layouts
self.topLayout.addWidget(self.label)
self.topLayout.addStretch()
self.buttonLayout.addWidget(self.pb_addline)
self.buttonLayout.addWidget(self.pb_rmline)
self.buttonLayout.addWidget(self.pb_addfile)
self.buttonLayout.addStretch()
self.buttonLayout.addWidget(self.pb_save)
self.buttonLayout.addWidget(self.pb_cancel)
self.layout.addLayout(self.topLayout)
self.layout.addWidget(self.table)
self.layout.addLayout(self.buttonLayout)
# connect the buttons and the table
self.pb_save.clicked.connect(self.save)
self.pb_cancel.clicked.connect(self.close)
self.pb_addline.clicked.connect(self.addLine)
self.pb_rmline.clicked.connect(self.removeLine)
self.pb_addfile.clicked.connect(self.addFiles)
self.table.cellChanged.connect(self.reload)
self.table.dropped.connect(self.addFiles)
self.pb_save.setDefault(True)
self.pb_save.setFocus()
self.exec_()
def addLine(self):
""" adds one row to the table """
index = self.table.rowCount()
self.table.insertRow(index)
self.table.setItem(index, 0, QtWidgets.QTableWidgetItem(""))
self.fillCheckItem(index, "")
self.reload()
def addItem(self, index, path):
""" adds an item in a given row to the table """
filepath, ext = os.path.splitext(path)
if ext == ".h5" or ext ==".hdf5":
self.table.setItem(index, 0, QtWidgets.QTableWidgetItem(path))
return True
else:
return False
def removeLine(self):
for index in reversed(self.table.selectedIndexes()):
self.table.removeRow(index.row())
def addFiles(self, files=[]):
""" gets called when add File button gets clicked """
if not files:
fileDialog = self.CustomFileDialog()
execAccepted = fileDialog.exec_()
files, openClicked = fileDialog.getReturnState()
if not (execAccepted or openClicked):
return
def append(path):
nameFound = False
if len(self.table.selectedIndexes()) > 0:
item = self.table.selectedIndexes()[0]
if len(self.table.item(item.row(), 0).text()) == 0:
nameFound = True
self.addItem(item.row(), path)
if not nameFound:
emptyFound = False
for index in range(self.table.rowCount()):
item = self.table.item(index, 0)
if item.text() == "":
emptyFound = True
break
if not emptyFound:
index = self.table.rowCount()
self.addLine()
self.addItem(index, path)
for path in files:
if os.path.isfile(path):
append(path)
elif os.path.isdir(path):
for root, dirs, files in os.walk(path):
for name in reversed(files):
filepath, extension = os.path.splitext(os.path.join(root, name))
if extension == ".h5" or extension == ".hdf5":
append(filepath+extension)
self.reload()
def checkIfExists(self, path):
return os.path.exists(path)
def fillCheckItem(self, index, line):
""" checks wether a path is valid or missing """
if self.checkIfExists(line):
ext = os.path.splitext(line)[1]
if ext == ".h5" or ext == ".hdf5":
icon = self.icon_exists
label = "found"
else:
icon = self.icon_missing
label = "invalid"
else:
icon = self.icon_missing
label = "empty" if line == "" else "missing"
self.table.setItem(index, 1, QtWidgets.QTableWidgetItem(icon, label))
item = self.table.item(index, 1)
item.setFlags(Qt.NoItemFlags)
def reload(self, row=0, column=0):
""" reloads all path avalabilities """
if column == 0:
for index in range(self.table.rowCount()):
item = self.table.item(index, 0)
try:
self.fillCheckItem(index, item.text())
except AttributeError:
self.table.setItem(index, 1, QtWidgets.QTableWidgetItem(self.icon_missing, "missing"))
def save(self):
if self.path:
txtfile = open(self.path, "w+")
for index in range(self.table.rowCount()):
line = self.table.item(index, 0).text()
if self.checkIfExists(line):
txtfile.write(line+"\n")
txtfile.close()
self.accept()
else:
path, type = QtWidgets.QFileDialog.getSaveFileName(self, "Select a new Filename", QtCore.QDir.homePath(), "HDF5TXT (*.txt)")
if len(path) > 0:
path, extension = os.path.splitext(path)
path = path + ".txt" # ensure .txt is the suffix
self.path = path
txtfile = open(path, "w+")
for index in range(self.table.rowCount()):
line = self.table.item(index, 0).text()
if self.checkIfExists(line):
txtfile.write(line+"\n")
txtfile.close()
self.accept()
def keyReleaseEvent(self, QKeyEvent):
if QKeyEvent.key() == QtCore.Qt.Key_Delete:
self.removeLine()
class CustomTable(QtWidgets.QTableWidget):
dropped = pyqtSignal(list)
def __init__(self, parent=None):
QtWidgets.QWidget.__init__(self, parent)
self.setAcceptDrops(True)
def dragEnterEvent(self, event):
if event.mimeData().hasUrls:
event.accept()
else:
event.ignore()
def dragMoveEvent(self, event):
if event.mimeData().hasUrls:
event.accept()
else:
event.ignore()
def dropEvent(self, event):
if event.mimeData().hasUrls:
event.setDropAction(QtCore.Qt.CopyAction)
event.accept()
links = []
for url in event.mimeData().urls():
links.append(str(url.toLocalFile()))
self.dropped.emit(links)
else:
event.ignore()
class CustomFileDialog(QtWidgets.QFileDialog):
""" to get Folders OR Files"""
def __init__(self, parent=None):
QtWidgets.QFileDialog.__init__(self, parent)
self.setWindowTitle("Select a Folder or hdf5 files")
self.setFileMode(QtWidgets.QFileDialog.ExistingFiles)
self.setNameFilter("HDF5 (*.h5 *.hdf5);; Directories")
buttons = self.findChildren(QtWidgets.QPushButton)
self.openBtn = [x for x in buttons if 'open' in str(x.text()).lower()][0]
self.openBtn.clicked.disconnect()
self.cancelBtn = [x for x in buttons if 'cancel' in str(x.text().lower())][0]
self.cancelBtn.clicked.disconnect()
# TODO: Open Button Label could be changed, LookIn however doesnt work yet
# self.setLabelText(QtWidgets.QFileDialog.Accept, "Choose")
# self.setLabelText(QtWidgets.QFileDialog.LookIn, "Choose")
self.openBtn.clicked.connect(self.openClick)
self.cancelBtn.clicked.connect(self.close)
self.openClicked = 0
def openClick(self):
self.openClicked = 1
self.hide()
def getReturnState(self):
return self.selectedFiles(), self.openClicked
Barista/gui/main_window/__init__.py 0000664 ڊ0011117 00000000000 13265130053 017562 0 ustar klemms o0cvpr Barista/gui/main_window/default_actions.py 0000664 ڊ0011117 00000125625 13265130053 021214 0 ustar klemms o0cvpr import json
import sys
import os
import re
import tempfile
import traceback
import backend.barista.caffe_versions as caffeVersions
from PyQt5 import QtWidgets, QtGui, QtCore
from PyQt5.QtCore import pyqtSignal
from PyQt5.QtGui import QPixmap, QIcon, QTransform
from PyQt5.QtWidgets import QStyle, QMessageBox
from backend.barista.constraints.session_run.training import checkMinimumTrainingRequirements
from backend.barista.project import Project, dirIsProject
from backend.barista.session.client_session import ClientSession
from backend.barista.session.session_utils import State
from backend.barista.utils.logger import Log
from backend.barista.utils.settings import applicationQSetting
from backend.caffe.proto_info import UnknownLayerTypeException
from backend.caffe import loader
from backend.caffe import saver
from gui.deployment_dialog import DeploymentDialog
from gui.export_solver_dialog import ExportSolverDialog
from gui.gui_util import getNewProjectDir, getOpenProjectDir
from gui.change_caffe_version_dialog import CaffeVersionDialog
class RecentlyData(QtCore.QObject):
""" Model for Recently-Menu """
recentlyChanged = pyqtSignal()
def __init__(self, parent=None, maxHistory=5):
super(RecentlyData, self).__init__(parent)
self._recently = []
self._maxHistory = maxHistory
def addRecently(self, text, data):
""" Add an entry.
It will contains the given data but show the given text.
"""
# Remove if already exists (because of order)
if [text, data] in self._recently:
self._recently.remove([text, data])
self._recently.append([text, data])
# Delete oldest if needed
if len(self._recently) >= self._maxHistory:
self._recently = self._recently[-self._maxHistory:]
self.recentlyChanged.emit()
def setRecently(self, recentlylist):
""" Set entries from a list.
recentlylist should look like [[text1,data1],[text2,data2],...]
"""
if type(recentlylist) != list:
return
self._recently = recentlylist
if len(self._recently) >= self._maxHistory:
self._recently = self._recently[-self._maxHistory:]
self.recentlyChanged.emit()
def getRecently(self):
""" Returns the list of recent entries in form of
[[text1, data1],[text2,data2],...]
"""
return self._recently
def getMostRecent(self):
""" returns: Most recent [text, data] tuple or None (if there are no recent elements)
"""
try:
return self._recently[-1]
except IndexError:
return None
def recentlyFromSerialization(self, string):
""" Deserialize recent entries from a json-string """
self.setRecently(json.loads(string))
def serializeRecently(self):
""" Serialize recent entries into a json-string and return it"""
return json.dumps(self._recently)
class RecentMenu(QtWidgets.QMenu):
""" QMenu showing recent files with a clear entry """
recentlySelected = pyqtSignal(object)
def __init__(self, name, recentlydata, parent=None):
super(RecentMenu, self).__init__(name, parent)
self._data = recentlydata # type: RecentlyData
self._data.recentlyChanged.connect(self.__rebuildMenu)
self.__rebuildMenu()
def __rebuildMenu(self):
self.clear()
recently = self._data.getRecently()
for i in range(0, len(recently)):
# Build Recently-Menu Reverse
(text, data) = recently[-i - 1]
validPath = os.path.exists(text)
if not validPath:
text = '(?) ' + text
try:
action = self.addAction("{}. {}".format(i, text))
except UnicodeEncodeError:
action = self.addAction("{}. (?)".format(i))
action.setEnabled(False)
else:
if not validPath:
action.setEnabled(False)
action.setData(data)
action.triggered.connect(self.__recentlyFound)
self.addSeparator()
clearAction = self.addAction(self.tr("Clear"))
clearAction.triggered.connect(lambda: self._data.setRecently([]))
def __recentlyFound(self):
sender = self.sender()
data = sender.data()
self.recentlySelected.emit(data)
class DefaultActions(QtCore.QObject):
""" DefaultActions implements function which are used by the user.
It can expose this function by building menus and toolbars.
"""
projectChanged = pyqtSignal(object)
parentChanged = pyqtSignal(object)
netManagerChanged = pyqtSignal(object)
viewManagerChanged = pyqtSignal(object)
quitTriggered = pyqtSignal()
def __init__(self, parentWdg=None):
""" Initialize DefaultActions.
parentWdg will be used for dialogs to block interaction to it if wanted.
"""
super(DefaultActions, self).__init__()
self._parent = parentWdg
self._netManager = None
self._viewManager = None
self._scene = None
self._editor = None
self._editMode = None
self._project = None
self._versionManager = None
self._inputManager = None
self._hostManager = None
self._solverDialog = None
self._deploymentDialog = None
# Build all actions
self.openNetAction = self._buildAction(self.tr("&Import Network"), None, lambda: self.openNetDialog())
self.openSolverAction = self._buildAction(self.tr("Import Solver"), None, lambda: self.openSolverDialog())
self.saveNetAction = self._buildAction(self.tr("&Export Network"), None, lambda: self.saveNetDialog())
self.saveSolverAction = self._buildAction("&Export Solver", None, lambda: self.saveSolverDialog())
self.exitAction = self._buildAction(self.tr("&Exit"), "Ctrl+Q", self.quitTriggered, None,
self.tr("Close Application"))
self.undoAction = self._buildAction(self.tr("Undo"), "Ctrl+Z", lambda: self._netManager.undo(),
icon="resources/leftArrow.png")
self.redoAction = self._buildAction(self.tr("Redo"), "Shift+Ctrl+Z", lambda: self._netManager.redo(),
icon="resources/rightArrow.png")
self.editNetAction = self._buildAction(self.tr("Edit &Network as Prototxt"), "Ctrl+E", self.openNetEditor)
self.editSolverAction = self._buildAction(self.tr("Edit &Solver as Prototxt"), "Shift+Ctrl+E",
self.openSolverEditor)
self.viewInputManager = self._buildAction(self.tr("&Input Manager"), "Ctrl+I", self.openInputManager, icon="resources/input_manager.png")
self.viewHostManager = self._buildAction(self.tr("&Host Manager"), "Ctrl+H", self.openHostManager, icon="resources/host_manager.png")
self.viewCaffeVersionManager = self._buildAction(self.tr("&Caffe Version Manager"), "Ctrl+M", self.changeCaffeVersion, icon="resources/CVM.png")
self.loadDefaultViewAction = self._buildAction(self.tr("&Load Default View"), None, self._loadDefaultView)
self.sortSceneAction = self._buildAction(self.tr("Arrange Layers Horizontally"), None, self._sortScene)
self.sortSceneActionVertical = self._buildAction(self.tr("Arrange Layers Vertically"), None, self._sortSceneVertical)
self.newProjectAction = self._buildAction(self.tr("New Project"), "Ctrl+N", lambda: self.newProjectDialog(),
icon="resources/newDocument.png")
self.loadProjectAction = self._buildAction(self.tr("Load Project"), "Ctrl+O", lambda: self.openProjectDialog(),
icon="resources/openDocument.png")
self.saveProjectAction = self._buildAction(self.tr("Save Project"), "Ctrl+S", lambda: self.saveProject(),
icon="resources/saveProject.png")
self.deployAction = self._buildAction(self.tr("Deploy and export"), None,
lambda: self.openDeploymentDialog())
# Build menus
self.dockMenu = QtWidgets.QMenu("Docks", self._parent)
self.toolBarMenu = QtWidgets.QMenu("Toolbars", self._parent)
# Recently-Menu
self.recentNetData = RecentlyData()
self.recentSolverData = RecentlyData()
self.recentProjectsData = RecentlyData()
self._loadActionSettings()
self.recentNetData.recentlyChanged.connect(self._saveActionSettings)
self.recentSolverData.recentlyChanged.connect(self._saveActionSettings)
self.recentProjectsData.recentlyChanged.connect(self._saveActionSettings)
self.recentNetMenu = None
self.recentSolverMenu = None
# Actions for the active session
self.startSessionAction = self._buildAction("Start Session", "F5",
lambda: self._viewManager.getSessionsDock().onStart(),
icon=QIcon(QPixmap('resources/start.png')))
self.pauseSessionAction = self._buildAction("Pause Session", "F6",
lambda: self._viewManager.getSessionsDock().onPause(),
icon=QIcon(QPixmap('resources/pause.png')))
self.snapshotAction = self._buildAction("Create Snapshot", "F8",
lambda: self._viewManager.getSessionsDock().onSnap(),
icon=QIcon(QPixmap('resources/snap.png')))
def projectHelper(p):
if self._versionManager and p:
self._versionManager.updateProject(p)
self.projectChanged.connect(projectHelper)
def setParent(self, parent):
""" Sets the parent widget.
It is primary used to block interaction when opening a dialog
"""
if parent == self._parent: return
# oldp = self._parent
self._parent = parent
self.parentChanged.emit(parent)
# Recent-Data
def _loadActionSettings(self):
settings = applicationQSetting()
settings.beginGroup("Actions")
self.recentNetData.recentlyFromSerialization(settings.value("recent_nets", "[]"))
self.recentSolverData.recentlyFromSerialization(settings.value("recent_solvers", "[]"))
self.recentProjectsData.recentlyFromSerialization(settings.value("recent_projects", "[]"))
settings.endGroup()
def _saveActionSettings(self):
settings = applicationQSetting()
settings.beginGroup("Actions")
settings.setValue("recent_nets", self.recentNetData.serializeRecently())
settings.setValue("recent_solvers", self.recentSolverData.serializeRecently())
settings.setValue("recent_projects", self.recentProjectsData.serializeRecently())
settings.endGroup()
# Recent-Gui-Menu
def buildRecentNetMenu(self, name="Recent Nets", parent=None):
return self.__buildRecentMenu(name, self.recentNetData, parent, self.openNet)
def buildRecentSolverMenu(self, name="Recent Solvers", parent=None):
return self.__buildRecentMenu(name, self.recentSolverData, parent, self.openSolver)
def buildRecentProjectMenu(self, name="Recent Projects", parent=None, callback=None):
if callback is None:
callback = lambda directory: self.setProjectAskUserForSave(directory, parent)
return self.__buildRecentMenu(name, self.recentProjectsData, parent, callback)
def __buildRecentMenu(self, name, data, parent, action):
if parent is None:
parent = self._parent
res = RecentMenu(name, data, parent)
res.recentlySelected.connect(action)
return res
# Update Action-State
def updateActionState(self):
""" Update Actions (setup enabled e.g.)"""
self._updateUndoRedoState()
self.saveProjectAction.setEnabled(not self._project is None)
def _updateUndoRedoState(self):
if self._netManager:
self.undoAction.setEnabled(self._netManager.canUndo())
self.redoAction.setEnabled(self._netManager.canRedo())
# Build-Gui-Elements
def buildMenuBar(self, parent=None):
""" Build a menubar with all available actions. Set its parent to the given one. """
if parent is None:
parent = self._parent
bar = QtWidgets.QMenuBar(self._parent)
fileMenu = bar.addMenu(self.tr("&File")) # type: QtWidgets.QMenu
fileMenu.addAction(self.newProjectAction)
fileMenu.addAction(self.loadProjectAction)
fileMenu.addMenu(self.buildRecentProjectMenu(parent=parent))
fileMenu.addAction(self.saveProjectAction)
fileMenu.addSeparator()
fileMenu.addAction(self.openNetAction)
fileMenu.addAction(self.openSolverAction)
fileMenu.addAction(self.saveNetAction)
fileMenu.addAction(self.saveSolverAction)
fileMenu.addSeparator()
self.recentNetMenu = self.buildRecentNetMenu(parent=parent)
fileMenu.addMenu(self.recentNetMenu)
self.recentSolverMenu = self.buildRecentSolverMenu(parent=parent)
fileMenu.addMenu(self.recentSolverMenu)
fileMenu.addSeparator()
fileMenu.addAction(self.exitAction)
editMenu = bar.addMenu(self.tr("&Edit"))
editMenu.addAction(self.undoAction)
editMenu.addAction(self.redoAction)
editMenu.addSeparator()
editMenu.addAction(self.sortSceneAction)
editMenu.addAction(self.sortSceneActionVertical)
editMenu.addSeparator()
editMenu.addAction(self.editNetAction)
editMenu.addAction(self.editSolverAction)
toolsMenu = bar.addMenu(self.tr("&Tools"))
toolsMenu.addAction(self.viewCaffeVersionManager)
toolsMenu.addAction(self.viewInputManager)
toolsMenu.addAction(self.viewHostManager)
viewMenu = bar.addMenu(self.tr("&View"))
viewMenu.addMenu(self.dockMenu)
viewMenu.addMenu(self.toolBarMenu)
viewMenu.addSeparator()
viewMenu.addAction(self.loadDefaultViewAction)
deployMenu = bar.addMenu(self.tr("&Deployment"))
deployMenu.addAction(self.deployAction)
viewMenu.aboutToShow.connect(lambda: self.triggerViewMenu())
return bar
def triggerViewMenu(self):
""" Update dock visibility when ViewMenu was clicked """
self._viewManager.onVisibilityChange()
def addDockMenuEntry(self, dockwdg, title=None):
""" Add a new menu entry for the dockwidget dockwdg """
if title is None:
title = dockwdg.windowTitle()
action = self.dockMenu.addAction(title)
action.setCheckable(True)
action.setChecked(dockwdg.isVisible())
action.triggered.connect(lambda: self._viewManager.dockAction(title))
dockwdg.visibilityChanged.connect(lambda vis: action.setChecked(vis))
return action
def addToolBarEntry(self, toolbar):
""" Add a new menu entry for the visibility of the given toolbar """
action = self.toolBarMenu.addAction(toolbar.windowTitle())
action.setCheckable(True)
action.setChecked(toolbar.isVisible())
action.triggered.connect(toolbar.setVisible)
toolbar.visibilityChanged.connect(action.setChecked)
return toolbar
def buildFileToolbar(self, title="File"):
""" Build toolbar with default actions for file"""
toolbar = QtWidgets.QToolBar(title)
toolbar.setObjectName("FileToolbar")
toolbar.addAction(self.newProjectAction)
toolbar.addAction(self.loadProjectAction)
toolbar.addAction(self.saveProjectAction)
toolbar.addAction(self.viewInputManager)
toolbar.addAction(self.viewHostManager)
toolbar.addAction(self.viewCaffeVersionManager)
return toolbar
def buildEditToolbar(self, title="Edit"):
""" Build toolbar with default actions for edit"""
toolbar = QtWidgets.QToolBar(title)
toolbar.setObjectName("EditToolbar")
toolbar.addAction(self.undoAction)
toolbar.addAction(self.redoAction)
return toolbar
def buildRunToolbar(self, title="Active Job"):
""" Build toolbar with default actions for edit"""
toolbar = QtWidgets.QToolBar(title)
toolbar.setObjectName("RunToolbar")
toolbar.addAction(self.startSessionAction)
toolbar.addAction(self.pauseSessionAction)
toolbar.addAction(self.snapshotAction)
return toolbar
def _buildAction(self, text, shortcut, fun, icon=None, statustip=None, standardPixmap=None):
""" Build QAction with given parameter """
res = QtWidgets.QAction(text, self._parent)
if standardPixmap:
icon = QtWidgets.qApp.style().standardIcon(standardPixmap)
if icon:
if type(icon) == str:
icon = QtGui.QIcon(icon)
res.setIcon(icon)
if statustip:
res.setStatusTip(statustip)
if shortcut:
res.setShortcut(shortcut)
if fun:
res.triggered.connect(fun)
return res
# Project
def checkRunningSessions(self):
""" Check if there are running sessions and if thats the case ask the
user to pause them.
"""
if self._project:
if self._project.hasRunningSessions():
QM = QtWidgets.QMessageBox
answer = QM.question(self._parent, self.tr("Pause sessions?"), self.tr(
"There are running sessions. To proceed all sessions must be paused. Do you want to pause them?"),
QM.Yes | QM.Cancel, QM.Cancel)
if answer == QM.Yes:
self._project.pauseRunningSessions()
return True
if answer == QM.Cancel:
return False
def checkSessionState(self, session, showWarnings):
valid = checkMinimumTrainingRequirements(session=session, parentGui=self._viewManager.sessionController.sessionGui, reportToUser=showWarnings)
if valid:
session.setState(session.getState())
else:
if not session.getState() == State.NOTCONNECTED:
session.setState(State.INVALID)
def setProjectAskUserForSave(self, project_dir, parent=None):
""" Load the project from the given dir.
The user will be asked for saving the current project if
there are some unsaved changes.
"""
try:
if self.checkRunningSessions() is False:
return
if os.path.exists(project_dir) is False:
if self.askRecreateProject(parent):
os.makedirs(project_dir)
else:
return
self.recentProjectsData.addRecently(project_dir, project_dir)
project = Project(project_dir)
if parent is None:
parent = self._parent
if self.askUserAndSave(parent):
self.setProject(project)
except:
self.errorMsg("Error while trying to use an existing project.", "set_project")
return False
def setProject(self, project):
""" Set the project.
It will be loaded automatically in gui if networkmanager is set.
"""
try:
if self._project == project: return
# Register project as (most) recently used
self.recentProjectsData.addRecently(project.projectRootDir, project.projectRootDir)
self.removeProjectFromPlotter(self._project)
self._project = project
self._inputManager.clearInputManager()
self._viewManager.setProject(self._project)
self.updateActionState()
self.__propagateProjectToNetworkManager()
self._project.newSession.connect(self.registerNewSession)
self._project.deleteSession.connect(self.disconnectSession)
activeSID = self._project.getActiveSID()
if activeSID:
self.registerNewSession(activeSID)
self.selectSessionInList(activeSID)
self.projectChanged.emit(project)
except:
self.errorMsg("Error while trying to use an existing project.", "set_project")
return False
def removeProjectFromPlotter(self, project):
""" Removes the given project from the plotter.
"""
if project:
for session in project.getSessions().values():
self._viewManager.getPlotter().removeLog(str(session.getLogId()), str(session.getLogFileName(True)))
def selectSessionInList(self, SID):
self._viewManager.getSessionsDock().setSelectedSID(SID)
def registerNewSession(self, SID):
self._project.getSession(SID).stateDictChanged.connect(self.checkSessionState)
def disconnectSession(self, SID):
try:
self._project.getSession(SID).stateDictChanged.disconnect()
except TypeError:
# will be raised if disconnect failed
# TODO: Not sure if this is the right way to do it. This happens if a non-active session is deleted.
pass
def changeSession(self, newSID, oldSID=None):
"""Changes the active session within one project.
The current State of the netManager is saved to the old session. The state of the new session is loaded to the netManager."""
self.storeSessionState(SID=oldSID, stateDict=None)
self._project.setActiveSID(newSID)
if self._project.getActiveSID() == newSID:
self.loadSessionState(SID=newSID)
else:
Log.log("New Session " + str(newSID) + " could not be set. Valid SIDs are: " +
", ".join([str(id) for id in self._project.getValidSIDs()]),
self._viewManager.sessionController.getCallerId())
return
def storeSessionState(self, SID=None, stateDict=None):
"""Stores the stateDict to a session.
If no SID is provided, the currently active session is used.
If no dictionary is provided, the current netManager state is used."""
dict = stateDict
if not dict:
dict = self._netManager.getStateDictionary()
_SID = SID
if not _SID:
_SID = self._project.getActiveSID()
elif _SID not in self._project.getValidSIDs():
Log.log("Could not store state to SID ", _SID, ". Valid SIDs are:" +
", ".join([str(id) for id in self._project.getValidSIDs()]),
self._viewManager.sessionController.getCallerId())
self._project.getSession(_SID).setStateDict(dict)
return
def loadSessionState(self, SID=None):
"""Loades the state from a session to the netManager.
If no Session is provided, the currently active session is used."""
_SID = SID
if not _SID:
_SID = self._project.getActiveSID()
if _SID in self._project.getValidSIDs():
if self._project.getSession(_SID).hasStateDict():
dict = self._project.getSession(_SID).state_dictionary
net = None
pos = None
sel = None
if "network" in dict:
net = dict["network"]
elif ("layerOrder" in dict) and ("layers" in dict):
net = dict
if "position" in dict:
pos = dict["position"]
if "selection" in dict:
sel = dict["selection"]
if net:
self._netManager.setStateDictionary(dictionary=dict, clearHistory=True)
else:
Log.log("Could not load state from Session," + str(_SID) + " is this an old Project?",
self._viewManager.sessionController.getCallerId())
else:
# TODO: write proper Error to log
print("Could not load state from session", _SID)
Log.log("Could not load state from session " + str(_SID),
self._viewManager.sessionController.getCallerId())
return
def __propagateProjectToNetworkManager(self):
""" Setup the current Project in NetworkManager """
if self._project is None:
return
if self._netManager is None:
return
state = self._project.getActiveSession().state_dictionary
inputdb = self._project.getInputManagerState()
transform = self._project.getViewTransform()
self._inputManager.importdb(inputdb)
self._netManager.setStateDictionary(state, clearHistory=True)
self._netManager.resetModifiedFlag()
if transform is None:
return
self._netManager.nodeEditor.getView().setTransform(
QTransform(transform[0], transform[1], transform[2], transform[3], transform[4], transform[5]))
# method called when creating a new project, either from start window
# or from project menu bar
def newProjectDialog(self, dir="", parent=None):
""" Open a new dialog for selecting a new project.
Parent is the parent for the dialogs. If None the set parent in this instance is used.
Returns true if successfully selecting a project dir, false otherwise.
"""
try:
if parent is None: parent = self._parent
if not self.askUserAndSave(parent):
return
# directory alreay selected?
if dir == "":
dir = getNewProjectDir(self, parent)
# directory valid? if not abort
if len(dir) == 0:
return
if self.checkRunningSessions() is False:
return False
# Create project
os.mkdir(dir)
self.setProject(Project(dir))
self.recentProjectsData.addRecently(dir, dir)
self.saveProject()
return True
except:
self.errorMsg("Error while trying to create a new project.", "create_project")
return False
def openProjectDialog(self, parent=None):
""" Open a new dialog for selecting an existing project.
Parent is the parent for the dialog. If None the set parent in this instance is used.
Returns true if successfully selecting a project, false otherwise.
"""
try:
if parent is None: parent = self._parent
if not self.askUserAndSave(parent):
return False
dir = getOpenProjectDir(self, parent)
if len(dir) == 0:
return False
if self.checkRunningSessions() is False:
return False
self.recentProjectsData.addRecently(dir, dir)
proj = Project(dir)
proj_old = self._project
self.setProject(proj)
if(proj.getCaffeVersion() != proj_old.getCaffeVersion()):
msgBox = QMessageBox(QMessageBox.Warning, "Warning", "Please restart Barista client for changes to apply, otherwise Barista may be unstable!")
msgBox.addButton("Ok", QMessageBox.NoRole)
msgBox.addButton("Restart now", QMessageBox.YesRole)
if msgBox.exec_() == 1:
self.restart()
return True
except UnknownLayerTypeException as e:
caffeVersionDialog = CaffeVersionDialog(e, dir, self)
caffeVersionDialog.exec_()
return False
except:
self.errorMsg("Error while trying to open an existing project.", "open_project")
return False
def restart(self, projDir=""):
"""Save the actual project and restart barista"""
if not projDir:
self.saveProject()
"""Determine arguments for the restart.
Catch the old -o/--open with regex1 and allow every other old arguments with regex2"""
parameter = []
regex1=re.compile("(-o)|(--open).*")
regex2=re.compile("([-.]).*")
for argument in sys.argv:
if not regex1.match(argument) and regex2.match(argument):
parameter.append(argument)
if not projDir:
parameter.append(str("-o"+self.getProjectPath()))
else:
parameter.append(str("-o"+projDir))
os.execl(sys.executable, sys.executable, *parameter)
def saveProject(self):
""" Save the project in the directory it lives """
try:
if self._project is None or self._netManager is None:
return False
dict = self._netManager.getStateDictionary()
inputs = self._inputManager.getDBDict()
transform = self._netManager.nodeEditor.getView().transform()
self._project.saveProject(dict, inputs, transform)
self._netManager.resetModifiedFlag()
return True
except:
self.errorMsg("Error while trying to save the project.", "save_project")
return False
def getProjectPath(self):
"""returns the project path for external classes"""
if self._project:
return self._project.projectRootDir
return None
# Network Manager
def saveNetDialog(self, parent=None):
""" Open a save dialog and saves the network.
Parent is the parent for the dialog. If None the set parent in this instance is used.
Returns True if successfully selecting an place to save, False otherwise.
"""
try:
if parent is None: parent = self._parent
if self._netManager is None:
return False
filename, _ = QtWidgets.QFileDialog.getSaveFileName(parent, self.tr("Export Network"))
if len(filename) == 0:
return False
self._netManager.saveNetworkToFile(filename)
self.recentNetData.addRecently(filename, filename)
return True
except:
self.errorMsg("Error while trying to export the network.", "network_export")
return False
def saveSolverDialog(self, parent=None):
""" Open a save dialog and saves the solver.
Parent is the parent for the dialog. If None the set parent in this instance is used.
Returns True if successfully selecting an place to save, False otherwise.
"""
try:
if parent is None:
parent = self._parent
if self._netManager is None:
return False
self._solverDialog = ExportSolverDialog(self._netManager.solver, self._netManager.network, parent, self)
self._solverDialog.show()
return True
except:
self.errorMsg("Error while trying to export the solver.", "solver_export")
return False
def openSolverDialog(self, parent=None):
""" Open a open dialog and load the solver.
Parent is the parent for the dialog. If None the set parent in this instance is used.
Returns True if successfully selecting a file to load, False otherwise.
"""
if parent is None: parent = self._parent
if self._netManager is None:
return False
filename, _ = QtWidgets.QFileDialog.getOpenFileName(parent, self.tr("Import Solver File"), "",
"Prototxt (*.prototxt) ;; All Files (*)")
if len(filename) == 0:
return False
try:
self.openSolver(filename)
return True
except loader.ParseException as ex:
QtWidgets.QMessageBox.critical(parent, self.tr("Error parsing prototxt"), str(ex))
return False
def openSolver(self, filename):
if not os.path.exists(filename):
QtWidgets.QMessageBox.critical(None, self.tr("File does not exists"),
self.tr("Cannot load {}. File does not exists.").format(filename))
return
# open solver
self._netManager.openSolverFromFile(filename)
self.recentSolverData.addRecently(filename, filename)
# check whether the loaded solver does contain a network specification, too
# (we need to load the solver once again, as the redundant network specification has been removed automatically)
with open(filename, 'r') as file:
solverOriginalString = file.read()
solverOriginalDictionary = loader.loadSolver(solverOriginalString)
solverRefersToNetFile = "net" in solverOriginalDictionary
solverIncludesNet = "net_param" in solverOriginalDictionary
# Theoretically, the solver definition might contain both: a file reference and an inline definition
# We do support this case, but as it will be very seldom, we do not create a new main_window handling both options.
# Instead we just show two separate windows after each other
if solverRefersToNetFile:
netPath = solverOriginalDictionary["net"]
# if the given path is not an absolute one..
if not os.path.isabs(netPath):
# it's either relative to the caffe root directory
netPathAlternative1 = os.path.join(caffeVersions.getVersionByName(self._project.getCaffeVersion()).getRootpath(), netPath)
# or relative to the solver file (e.g. for Barista sessions)
netPathAlternative2 = os.path.join(os.path.dirname(filename), netPath)
if netPathAlternative1 and os.path.isfile(netPathAlternative1):
netPath = netPathAlternative1
else:
netPath = netPathAlternative2
self._promptForAutoNetImport(netPath)
if solverIncludesNet:
# get the network definition as a string
netString = loader.extractNetFromSolver(solverOriginalString)
# save inline definition to temporary file, so we can use the same algorithm as above
tempFilePath = tempfile.NamedTemporaryFile(delete=False)
tempFilePath.write(netString)
tempFilePath.close()
self._promptForAutoNetImport(tempFilePath.name)
os.remove(tempFilePath.name)
def _promptForAutoNetImport(self, netPath):
"""Open a message box and ask the user if he/she wants to import the net definition given in netPath.
netPath contains a net definition that has been extracted from a loaded solver definition.
"""
# check whether the specified file does exist
if os.path.isfile(netPath):
# get the file content
with open(netPath, 'r') as file:
netPrototxt = file.read()
msgBox = QMessageBox()
msgBox.setWindowTitle("Barista")
msgBox.setText(self.tr("Solver definition contains a network reference."))
msgBox.setInformativeText(self.tr("Do you want to import the network, too?"))
msgBox.setDetailedText("File:\n{}\n\nNet definition:\n{}".format(netPath, netPrototxt))
msgBox.setStandardButtons(QMessageBox.Yes | QMessageBox.No)
msgBox.setDefaultButton(QMessageBox.Yes)
ret = msgBox.exec_()
if ret == QMessageBox.Yes:
self.openNet(netPath)
else:
callerId = Log.getCallerId('file_loader')
Log.log("Found network reference in loaded solver definition, but the file {} does not exist.".format(
netPath
), callerId)
def openNet(self, filename):
if not os.path.exists(filename):
QtWidgets.QMessageBox.critical(None, self.tr("File does not exists"),
self.tr("Cannot load {}. File does not exists.").format(filename))
return
self._netManager.openNetworkFromFile(filename)
self.recentNetData.addRecently(filename, filename)
def openNetDialog(self, parent=None):
""" Open a open dialog and load the network.
Parent is the parent for the dialog. If None the set parent in this instance is used.
Returns True if successfully selecting a file to load, False otherwise.
"""
if parent is None: parent = self._parent
if self._netManager is None:
return False
filename, _ = QtWidgets.QFileDialog.getOpenFileName(parent, self.tr("Import Network File"), "",
"Prototxt (*.prototxt) ;; All Files (*)")
if len(filename) == 0:
return False
try:
self.openNet(filename)
return True
except loader.ParseException as ex:
QtWidgets.QMessageBox.critical(parent, self.tr("Error parsing prototxt"), str(ex))
return False
def setNetworkManager(self, networkManager):
""" Set the Networkmanager the actions should work with"""
if networkManager == self._netManager: return
if self._netManager:
self._netManager.stateChanged.disconnect(self.updateActionState)
self._netManager = networkManager
if self._netManager:
self._netManager.stateChanged.connect(self.updateActionState)
self.updateActionState()
self.netManagerChanged.emit(networkManager)
# View Manager
def setViewManager(self, viewManager):
"""Set the ViewManager the actions should work with"""
if viewManager == self._viewManager: return
self._viewManager = viewManager
self.updateActionState()
self.viewManagerChanged.emit(viewManager)
def _loadDefaultView(self):
if self._viewManager is None:
return
self._viewManager.loadDefaultView()
def _sortScene(self):
self._netManager.nodeEditor.rearrangeNodes()
def _sortSceneVertical(self):
self._netManager.nodeEditor.rearrangeNodesVertical()
# SCENE
def setScene(self, scene):
self._scene = scene
self.updateActionState()
# EDITOR
def openNetEditor(self):
""" Open a Editor for editing the prototxt of the network """
if self._editor is None:
return False
self._editMode = "Network"
self._editor.setText(saver.saveNet(self._netManager.network))
self._editor.show()
return True
def openSolverEditor(self):
""" Open a Editor for editing the prototxt of the solver """
if self._editor is None:
return False
self._editMode = "Solver"
self._editor.setText(saver.saveSolver(self._netManager.solver))
self._editor.show()
return True
def openInputManager(self):
if self._inputManager is None:
return False
self._inputManager.updateListWidget()
self._inputManager.show()
return True
def openHostManager(self):
if self._hostManager is None:
return False
self._hostManager.show()
return True
def _editorChangedText(self, netproto):
try:
netproto = netproto.encode("utf-8")
if self._netManager:
if self._editMode == "Network":
self._netManager.openNetworkFromString(netproto, False)
self._netManager.checkForDuplicateNames()
if self._editMode == "Solver":
self._netManager.openSolverFromString(netproto)
self._project.getActiveSession().setMaxIteration(int(loader.loadSolver(netproto).get("max_iter", 1)))
except loader.ParseException as ex:
QtWidgets.QMessageBox.critical(self._parent, "Error parsing prototxt", str(ex))
def setEditor(self, editor):
"""Set the Editor the actions should work with"""
if self._editor:
self._editor.sgSave.disconnect(self._editorChangedText)
self._editor = editor
if self._editor:
self._editor.sgSave.connect(self._editorChangedText)
self.updateActionState()
def setInputManager(self, manager):
"""sets the input Manager"""
self._inputManager = manager
self.updateActionState()
def setHostManager(self, manager):
"""sets the host Manager"""
self._hostManager = manager
self.updateActionState()
def changeCaffeVersion(self):
if self._versionManager is None:
return False
self._versionManager.show()
def openDeploymentDialog(self):
"""Show a file selection main_window and export a deployment network to the selected location."""
self._deploymentDialog = DeploymentDialog(self._project.getSessions(), self._parent)
self._deploymentDialog.show()
return True
def setVersionManager(self, manager):
self._versionManager = manager
self.updateActionState()
def askUserAndSave(self, parent=None):
""" Ask the user for saving if something changes since last save.
If the user want to save, the project get saved.
If the user want to abort, return False else return True
"""
if parent is None:
parent = self._parent
if not self._netManager or not self._netManager.isModified():
return True
QM = QtWidgets.QMessageBox
answer = QM.question(parent, self.tr("Are you sure?"), self.tr("Do you want to save the changes you made?"),
QM.Yes | QM.No | QM.Abort, QM.Abort)
if answer == QM.Abort:
return False
if answer == QM.Yes:
self.saveProject()
return True
def askRecreateProject(self, parent=None):
""" Ask the user if he wants to create a new project in the directory.
"""
QM = QtWidgets.QMessageBox
answer = QM.question(parent, self.tr("Are you sure?"), self.tr(
"The directory does not exists. Do you want to create a new project in this directory?"), QM.Yes | QM.Abort,
QM.Abort)
if answer == QM.Yes:
self.saveProject()
return True
if answer == QM.Abort:
return False
return False
def errorMsg(self, errorMsg, loggerIdString=None, addStacktrace=True):
"""Show an error message in the Logger as well as in an additional GUI popup."""
# use the logger
if loggerIdString is not None:
callerId = Log.getCallerId(loggerIdString)
else:
callerId = None
Log.log(errorMsg, callerId)
# show message in the GUI
msgBox = QMessageBox()
msgBox.setWindowTitle("Barista - Error")
msgBox.setText(self.tr(errorMsg))
if addStacktrace:
stacktrace = traceback.format_exc()
msgBox.setDetailedText(stacktrace)
msgBox.setStandardButtons(QMessageBox.Ok)
_ = msgBox.exec_()
def disableEditing(self, disable):
""" Disable actions for loading a new net or solver. """
self.openNetAction.setDisabled(disable)
self.openSolverAction.setDisabled(disable)
if self.recentNetMenu:
self.recentNetMenu.setDisabled(disable)
if self.recentSolverMenu:
self.recentSolverMenu.setDisabled(disable)
Barista/gui/main_window/main_window.py 0000664 ڊ0011117 00000020472 13265130053 020355 0 ustar klemms o0cvpr # To change this license header, choose License Headers in Project Properties.
# To change this template file, choose Tools | Templates
# and open the template in the prototxt_editor.
from PyQt5 import QtWidgets, QtCore
from PyQt5.QtGui import QIcon
from PyQt5.QtCore import pyqtSignal
import gui.main_window.default_actions
import gui.main_window.status_bar
from backend.barista.session.session_utils import State
from backend.caffe import dict_helper
from gui.caffe_version_manager import caffe_version_manager
from gui.input_manager import input_manager
from gui.host_manager import host_manager
from gui.main_window import view_manager
from gui.main_window.node_editor.node_editor import NodeEditor
from gui.network_manager import network_manager
from gui.prototxt_editor import editor_widget
class UiMainWindow(QtWidgets.QMainWindow):
sessionSelectionChanged = pyqtSignal(object)
def __init__(self, actions=None, desktop = None):
QtWidgets.QMainWindow.__init__(self)
self.setUnifiedTitleAndToolBarOnMac(True)
self.disabled = False
#self.setGeometry(desktop.screenGeometry()) #fullscreen
self.setWindowTitle("Barista")
self.setWindowIcon(QIcon('./resources/icon.png'))
# Initialize menu and status bar
if actions:
self.actions = actions
self.actions.setParent(self)
else:
self.actions = gui.main_window.default_actions.DefaultActions(self)
self.editor = editor_widget.EditorWidget(parent=self)
self.editor.setWindowTitle("Barista Network Editor")
self.actions.setEditor(self.editor)
self.inManager = input_manager.InputManager(self)
self.actions.setInputManager(self.inManager)
self.hostManager = host_manager.HostManager(self)
self.actions.setHostManager(self.hostManager)
caffeVersionManager = caffe_version_manager.CaffeVersionManager(self.hostManager, self.actions)
caffeVersionManager.setWindowTitle("Caffe Version Manager")
self.actions.setVersionManager(caffeVersionManager)
self.setMenuBar(self.actions.buildMenuBar())
filetoolbar = self.actions.buildFileToolbar()
self.addToolBar(filetoolbar)
self.actions.addToolBarEntry(filetoolbar)
edittoolbar=self.actions.buildEditToolbar()
self.addToolBar(edittoolbar)
self.actions.addToolBarEntry(edittoolbar)
#runtoolbar=self.actions.buildRunToolbar()
#self.addToolBar(runtoolbar)
#self.actions.addToolBarEntry(runtoolbar)
# Create the view manager
self.viewManager = view_manager.ViewManager(self, self.actions)
self.actions.setViewManager(self.viewManager)
self.setStatusBar(gui.main_window.status_bar.UiStatusBar(self))
# Create the view
self.viewManager.createView()
# Create the node prototxt_editor as the central widget
self.nodeEditor = NodeEditor(self)
self.setCentralWidget(self.nodeEditor.getView())
self.actions.setScene(self.nodeEditor.getView().scene())
# Connect the node editor context menu.
self.nodeEditor.getView().arrangeVerticallyClicked.connect(self.actions.sortSceneActionVertical.trigger)
self.nodeEditor.getView().arrangeHorizontallyClicked.connect(self.actions.sortSceneAction.trigger)
# Create the network manager to save all layers and handle layer-related events
self.networkManager = network_manager.NetworkManager(self.viewManager.getActiveLayersDock(),
self.viewManager.getLayerPropertyDock(),
self.viewManager.getSolverPropertyDock,
self.nodeEditor)
# Link networkManager to menuBar
self.actions.setNetworkManager(self.networkManager)
# As a test, open the dummy network file
self.networkManager.setNetwork(dict_helper.bareNet("default"))
#connect all signals
self.actions.quitTriggered.connect(self.close)
self.actions.projectChanged.connect(self._onProjectChanged)
self.networkManager.modifiedChanged.connect(self._onNetworkChanged)
self.viewManager.sessionController.sessionSelectionChanged.connect(self._onSessionChanged)
self.networkManager.newStateData.connect(self._onStateDictChanged)
self.viewManager.sessionController.sessionStateRefreshed.connect(self.disableEditing)
self.rec_installEventfilter()
def eventFilter(self, source, event):
if event.type() == QtCore.QEvent.MouseButtonRelease and event.button() == QtCore.Qt.LeftButton:
self.viewManager.onVisibilityChange()
return super(UiMainWindow, self).eventFilter(source, event)
def rec_installEventfilter(self):
def recursive_set(parent):
for child in parent.findChildren(QtCore.QObject):
try:
child.installEventFilter(self)
except:
pass
recursive_set(child)
self.installEventFilter(self)
recursive_set(self)
def availableActions(self):
return self.actions
def showPathWarning(self):
self.warningBox = QtWidgets.QMessageBox()
self.warningBox.setText("caffe-path does not exists on your PC, please edit caffe-path in settings and restart the program")
self.warningBox.setIcon(2)
self.warningBox.show()
def closeEvent(self, event):
# check if there are running sessions and ask the user to pause them
if self.actions.checkRunningSessions() is False:
event.ignore()
return
# Save the current view
self.viewManager.saveView()
# Ask the user if he wants to save project changes if necessary.
if self.actions.askUserAndSave(self):
# We need to process events here, because when the project is saved
# an event is emitted. If this doesn't get processed, the main window
# will not be closed.
QtWidgets.QApplication.processEvents()
event.accept()
else:
# The user decided to abort the close event.
event.ignore()
def _onProjectChanged(self, project):
from backend.caffe.path_loader import PathLoader
self.setWindowTitle("Barista - {}".format(project.getProjectName()))
#reload the caffe information
self.actions._versionManager.updateProject(project)
# TODO: Those lines should not be commented, right?
# self.editor = editor_widget.EditorWidget(self)
# self.editor.setWindowTitle("Barista Network Editor")
# network_manager.helper.bareNet(project.getProjectName())
# self.viewManager.loadView()
def _onStateDictChanged(self, stateDict):
self.viewManager.project.getActiveSession().state_dictionary = stateDict
def _onSessionChanged(self):
activeSession = self.viewManager.project.getActiveSession()
self.networkManager.setState(activeSession.state_dictionary if activeSession is not None else None)
def _onNetworkChanged(self, changeSinceLastSave):
#update the status bar
self.statusBar().showModifiedFlag(changeSinceLastSave)
#update the current session
self.viewManager.project.setActiveSessionStateDict(self.networkManager.getStateDictionary())
def _onSolverChanged(self, changedItems, changedObject):
if "max_iter" in changedItems:
activeSession = self.viewManager.project.getActiveSession()
if not activeSession.isRemote():
activeSession.setMaxIteration(0)
self.viewManager.sessionController.sessionIterationChanged(activeSession)
def disableEditing(self, state):
""" Handle event on state change of a session and disable/enable editing.
"""
disable = False
if state in [State.RUNNING, State.PAUSED, State.FINISHED, State.NOTCONNECTED, State.FAILED]:
disable = True
self.actions.disableEditing(disable)
self.viewManager.disableEditing(disable)
self.nodeEditor.disableEditing(disable)
self.inManager.disableEditing(disable)
self.editor.disableEditing(disable)
self.show()
self.disabled = disable
def getDisabled(self):
return self.disabled
Barista/gui/main_window/status_bar.py 0000664 ڊ0011117 00000001132 13265130053 020201 0 ustar klemms o0cvpr # To change this license header, choose License Headers in Project Properties.
# To change this template file, choose Tools | Templates
# and open the template in the prototxt_editor.
from PyQt5 import QtWidgets
class UiStatusBar(QtWidgets.QStatusBar):
def __init__(self, parent=None):
QtWidgets.QStatusBar.__init__(self, parent)
self.modifyWdg = QtWidgets.QLabel()
self.addPermanentWidget(self.modifyWdg)
def showModifiedFlag(self, modified):
if modified:
self.modifyWdg.setText("Changes *")
else:
self.modifyWdg.setText("")
Barista/gui/main_window/view_manager.py 0000664 ڊ0011117 00000021605 13265130053 020505 0 ustar klemms o0cvpr from PyQt5.QtCore import Qt
from backend.barista.session.session_utils import State
from gui.main_window.docks import (
dock_available_layers,
dock_active_layers
)
from gui.main_window.docks.session import dock_session_list
from backend.barista.utils.settings import applicationQSetting
from gui.main_window.docks.dock_console import DockElementConsole
from gui.main_window.docks.plotter.qt5_plotter_gui import DockElementPlotter
from gui.main_window.docks.properties import dock_layer_properties, dock_solver_properties
from gui.main_window.docks.session.session_controller import SessionController
from gui.main_window.docks.weight_visualization.dock_weight_visualization import DockElementWeightPlotter
class ViewManager():
"""Management of the current view state of the main main_window"""
def __init__(self, mainWindow, actions):
# Store the variables for the class
self.mainWindow = mainWindow
self.actions = actions
self._defaultState = None
self._isDefaultStateSaved = False
self.dicDockWidgets = {}
self.dicDockActions = {}
self.sessionController = SessionController()
def _addDock(self, element, title):
"""Creates the Dock-Element element with title and mainwindow and add it to Menu"""
dock = element(self.mainWindow, title)
self.dicDockWidgets[title] = dock
self.dicDockActions[title] = self.actions.addDockMenuEntry(dock, title)
return dock
def createView(self):
"""Creates the docks of the application and loads the view settings"""
# Create the dock widgets and add the menu bar actions
self._addDock(dock_available_layers.DockElementLayers, 'Available Layers')
self._addDock(dock_active_layers.DockElementActivLayers, "Network Layers")
self._addDock(dock_session_list.DockElementSessions, "Sessions")
self._addDock(dock_layer_properties.LayerProperties, "Layer Properties")
self._addDock(dock_solver_properties.SolverProperties, "Solver Properties")
bottomDock=self._addDock(DockElementConsole, "Console")
bottomDock.setMinimumHeight(100)
plotterDock=self._addDock(DockElementPlotter, "Plotter")
plotterDock.setMinimumHeight(100)
weightPlotterDock=self._addDock(DockElementWeightPlotter, "Weight Visualization")
weightPlotterDock.setMinimumHeight(100)
self.sessionController.setWeightPlotter(weightPlotterDock)
self.sessionController.setPlotterGui(self.dicDockWidgets['Plotter'])
self.dicDockWidgets['Sessions'].setController(self.sessionController)
self.loadDefaultView()
# Try to load the view if the user has saved one
self.loadView()
def getLayersDock(self):
return self.dicDockWidgets['Available Layers']
def getActiveLayersDock(self):
return self.dicDockWidgets['Network Layers']
def getLayerPropertyDock(self):
return self.dicDockWidgets['Layer Properties']
def getSolverPropertyDock(self):
return self.dicDockWidgets['Solver Properties']
def getSessionsDock(self):
return self.dicDockWidgets['Sessions']
def dockAction(self, title):
'Shows or hides a dock based on input from menu bar action'
if self.dicDockWidgets[title].isHidden():
self.dicDockWidgets[title].show()
else:
self.dicDockWidgets[title].hide()
#reload the hooks in the view menu for the docks
self.onVisibilityChange()
def onVisibilityChange(self):
'Shows or hides a hook for the docks in the view menu'
# widget is only the title as string
for widget in self.dicDockWidgets:
if self.dicDockWidgets[widget].isVisible():
self.dicDockActions[widget].setChecked(True)
else:
self.dicDockActions[widget].setChecked(False)
def loadDefaultView(self):
'Loads the standard view settings'
if self._isDefaultStateSaved == False:
# Add layers dock
self.mainWindow.addDockWidget(Qt.LeftDockWidgetArea, self.dicDockWidgets['Available Layers'])
self.dicDockWidgets['Available Layers'].setFloating(False)
self.dicDockWidgets['Available Layers'].show()
# Add network layers dock
# self.mainWindow.addDockWidget(Qt.LeftDockWidgetArea, self.dicDockWidgets['Network Layers'])
self.mainWindow.tabifyDockWidget(self.dicDockWidgets['Available Layers'], self.dicDockWidgets['Network Layers'])
self.dicDockWidgets['Network Layers'].setFloating(False)
self.dicDockWidgets['Network Layers'].show()
self.dicDockActions['Network Layers'].setChecked(True)
self.dicDockWidgets['Available Layers'].raise_()
# Add running tasks dock
self.mainWindow.addDockWidget(Qt.RightDockWidgetArea, self.dicDockWidgets['Sessions'])
self.dicDockWidgets['Sessions'].setFloating(False)
self.dicDockWidgets['Sessions'].show()
self.dicDockActions['Sessions'].setChecked(True)
# Add solver properties dock
# self.mainWindow.addDockWidget(Qt.RightDockWidgetArea, self.dicDockWidgets['Solver Properties'])
self.mainWindow.tabifyDockWidget(self.dicDockWidgets['Sessions'], self.dicDockWidgets['Solver Properties'])
self.dicDockWidgets['Solver Properties'].setFloating(False)
self.dicDockWidgets['Solver Properties'].show()
self.dicDockActions['Solver Properties'].setChecked(True)
# Add layer properties dock
# self.mainWindow.addDockWidget(Qt.RightDockWidgetArea, self.dicDockWidgets['Layer Properties'])
self.mainWindow.tabifyDockWidget(self.dicDockWidgets['Solver Properties'], self.dicDockWidgets['Layer Properties'])
self.dicDockWidgets['Layer Properties'].setFloating(False)
self.dicDockWidgets['Layer Properties'].show()
self.dicDockActions['Layer Properties'].setChecked(True)
self.dicDockWidgets['Sessions'].raise_()
# Add Console dock
self.mainWindow.addDockWidget(Qt.BottomDockWidgetArea, self.dicDockWidgets['Console'])
self.dicDockWidgets['Console'].setFloating(False)
self.dicDockWidgets['Console'].show()
self.dicDockActions['Console'].setChecked(True)
# Add plotter dock
self.mainWindow.addDockWidget(Qt.BottomDockWidgetArea, self.dicDockWidgets['Plotter'])
self.dicDockWidgets['Plotter'].setFloating(False)
self.dicDockWidgets['Plotter'].show()
self.dicDockActions['Plotter'].setChecked(True)
# Add weight dock
# self.mainWindow.addDockWidget(Qt.BottomDockWidgetArea, self.dicDockWidgets['Weight Visualization'])
self.mainWindow.tabifyDockWidget(self.dicDockWidgets['Plotter'], self.dicDockWidgets['Weight Visualization'])
self.dicDockWidgets['Weight Visualization'].setFloating(False)
self.dicDockWidgets['Weight Visualization'].show()
self.dicDockActions['Weight Visualization'].setChecked(True)
self.dicDockWidgets['Plotter'].raise_()
# Save this state as default so that multiple clicks on the restore default view action do not cause changes
self._isDefaultStateSaved = True
self._defaultState = self.mainWindow.saveState()
else:
self.mainWindow.restoreState(self._defaultState)
self.onVisibilityChange()
def loadView(self):
'Loads the user-defined view settings from view_settings'
settings = applicationQSetting()
settings.beginGroup("ViewManager")
state = settings.value("guistate", None)
settings.endGroup()
if state:
self.mainWindow.restoreState(state)
self.onVisibilityChange()
def saveView(self):
'Saves the current view settings to view_settings file'
settings = applicationQSetting()
settings.beginGroup("ViewManager")
settings.setValue("guistate", self.mainWindow.saveState())
settings.endGroup()
def setProject(self, project):
""" Set the current project of the appliction.
"""
self.project = project
self.sessionController.setProject(project)
self.dicDockWidgets['Plotter'].setProject(project)
def getProject(self):
""" Return the current project.
"""
return self.project
def disableEditing(self, disable):
""" Disable editing on all docks.
"""
self.dicDockWidgets['Solver Properties'].disableEditing(disable)
self.dicDockWidgets['Layer Properties'].disableEditing(disable)
self.dicDockWidgets['Available Layers'].disableEditing(disable)
self.dicDockWidgets['Network Layers'].disableEditing(disable)
def getPlotter(self):
""" Return the plotter.
"""
return self.dicDockWidgets['Plotter']
Barista/gui/network_manager/__init__.py 0000664 ڊ0011117 00000000000 13265130053 020432 0 ustar klemms o0cvpr Barista/gui/network_manager/history_manager.py 0000664 ڊ0011117 00000006300 13265130053 022077 0 ustar klemms o0cvpr class HistoryManager():
def __init__(self, maxSize=100):
""" Initialize HistoryManager which saved maximal maxSize states.
The maxSize+1 insertion removes the first
"""
self.history = []
# Position is a value contains the index of the state in a continues way.
# That means even when the first entries get deleted because of maxSize
# self.position do not get decreased by this.
self.position = -1
# the index of the current position in history list
self.__idx = -1
# self.listLen contains maxSize
self.listLen = maxSize
# If this gets true, no other entry can be make in history
self.__lockHistory = False
def canUndo(self):
return self.__idx >0
def canRedo(self):
return self.__idx < len(self.history)-1
def undo(self):
if self.canUndo():
self.__idx -=1
self.position -= 1
def redo(self):
if self.canRedo():
self.__idx +=1
self.position += 1
def currentState(self):
""" Get latest state saved in history"""
if len(self.history) > 0:
return self.history[self.__idx]
else:
return None
def _insert(self,element):
""" Insert element at the current position """
# remove newer elements
del self.history[self.__idx + 1:]
# Remove the oldest element if there are too many elements
if self.__idx == self.listLen:
del self.history[0]
else:
self.__idx += 1
self.history.append(element)
self.position += 1
def lockHistory(self, fun):
""" Lock history for the whole method fun.
As a result no history can be inserted whil
fun is executed.
"""
if self.__lockHistory:
return
self.__lockHistory = True
fun()
self.__lockHistory = False
def insertFunc(self, fun):
""" Insert an element in history using the function fun.
While fun is working insertFunc is locked, so no other
element can be added in history.
As a result recursivly insertion of history is stopped.
The function fun will be called with in insertion function
which can be called to insert an element in history.
E.g.:
def createNewElement(text):
# Nothing happend, because insertFunc is locked
historymanager.insertFunc(lambda insertF: insertF(text))
return text
historymananger.insertFunc(lambda insertF: insertF(createNewElement("bla"))
# Only one string will be insert
When inserting a new state old state gets removed if the limit of
entries is reached.
Also states newer than the current one (e.g. by using undo())
get removed (so you can't do a redo() anymore)
"""
if self.__lockHistory:
return
self.__lockHistory = True
fun(self._insert)
self.__lockHistory = False
def clear(self):
self.position = -1
self.__idx = -1
self.history = []
Barista/gui/network_manager/layer_helper.py 0000664 ڊ0011117 00000020524 13265130053 021363 0 ustar klemms o0cvpr import os
class LayerHelper:
def __init__(self):
return
PHASE_TRAIN = u'TRAIN'
PHASE_TEST = u'TEST'
@staticmethod
def getLayerPhase(layer):
# TODO implement further rules to decide the phase of a layer (see issue #201)
# -> not only "include", but also "exclude"
# -> (include|exclude).min_level
# -> (include|exclude).max_level
# -> (include|exclude).stage
# -> (include|exclude).not_stage
# -> combination of multiple rules. e.g. include.phase == TRAIN and include.phase == TEST equals no value
# -> unclear: what does LayerParameter.phase (instead of LayerParameter.[include|exclude].phase) do?
if "include" in layer["parameters"]:
for item in layer["parameters"]["include"]:
if "phase" in item:
return item["phase"]
return ""
@staticmethod
def isLayerIncludedInTrainingPhase(layer):
"""Check whether this layer is included in the training phase."""
# TODO see TODO in LayerHelper.getLayerPhase().
return LayerHelper.getLayerPhase(layer) != LayerHelper.PHASE_TEST
@staticmethod
def isLayerInPlace(layer):
if "top" in layer["parameters"] and "bottom" in layer["parameters"]:
for bottomBlob in layer["parameters"]["bottom"]:
for topBlob in layer["parameters"]["top"]:
if bottomBlob == topBlob:
return True
return False
@staticmethod
def isLayerInPlaceForBlob(layer, blobName):
if "top" in layer["parameters"] and "bottom" in layer["parameters"]:
if not blobName in layer["parameters"]["top"]:
return False
if not blobName in layer["parameters"]["bottom"]:
return False
return True
else:
return False
@staticmethod
def getTopBlobIndex(layer, blobName):
if "top" in layer["parameters"]:
for index in range(0, len(layer["parameters"]["top"])):
if blobName == layer["parameters"]["top"][index]:
return index
return -1
@staticmethod
def buildConnectionList(network):
layerDict = network["layers"]
layerOrder = network["layerOrder"]
orderedLayersByBlob = OrderedLayersByBlob(layerDict, layerOrder)
blobOrder = orderedLayersByBlob.blobLayerDict
connections = list()
for layerID in layerOrder:
layer = layerDict[layerID]
LayerHelper.__addConnectionsForLayer(layer, layerID, blobOrder, connections, layerDict)
return connections
@staticmethod
def getLayersInfo(layertypes, network, dbpath):
""" Go over all layers in the network, search for layers of given types and return their id, name and a boolean
indicating if the layer's path matches the given dbpath.
"""
mylayer = []
for nid in network["layerOrder"]:
type = network["layers"][nid]["type"].name()
if type in layertypes:
params = network["layers"][nid]["parameters"]
layername = params["name"]
# since names can be ambiguous append the phase
if unicode("include") in params:
for incl in params["include"]:
if unicode("phase") in incl:
layername += " (" + incl["phase"] + ")"
same = False
dataParamType = "hdf5_data_param" if type == "HDF5Data" else "data_param"
if unicode(dataParamType) in params and unicode("source") in params[dataParamType]:
layerpath = params[dataParamType]["source"]
same = (os.path.realpath(dbpath) == os.path.realpath(layerpath))
mylayer.append([nid, layername, same])
return mylayer
@staticmethod
def setPathforType(id, path, type, network):
'''set the path and db type to layer basend on the ID'''
params = network["layers"][id]["parameters"]
if type in ["LMDB", "LEVELDB"]:
if "data_param" not in params:
params.giveProperty("data_param",{})
params["data_param"]["source"] = path
if type == "LMDB":
params["data_param"]["backend"] = u'LMDB'
if type == "LEVELDB":
params["data_param"]["backend"] = u'LEVELDB'
if type == "HDF5TXT":
if "hdf5_data_param" not in params:
params.giveProperty("hdf5_data_param",{})
params["hdf5_data_param"]["source"] = path
@staticmethod
def __addConnectionsForLayer(layer, layerID, blobOrder, connections, layerDict):
if "bottom" in layer["parameters"]:
bottoms = layer["parameters"]["bottom"]
for bottomIndex in range(0, len(bottoms)):
bottom = bottoms[bottomIndex]
if bottom in blobOrder:
topBlobList = blobOrder[bottom]
phaseDict = topBlobList[0]
if "" in phaseDict:
LayerHelper.__addConnection(phaseDict[""], layerID, bottomIndex, bottom, connections, layerDict)
LayerHelper.__removeBlobPhaseFromDict("", bottom, blobOrder)
elif LayerHelper.getLayerPhase(layer) == "":
for topPhase, topLayerID in phaseDict.iteritems():
LayerHelper.__addConnection(topLayerID, layerID, bottomIndex, bottom, connections, layerDict)
LayerHelper.__removeBlobPhaseFromDict(topPhase, bottom, blobOrder)
else:
phase = LayerHelper.getLayerPhase(layer)
for topPhase, topLayerID in phaseDict.iteritems():
if phase == topPhase:
LayerHelper.__addConnection(topLayerID, layerID, bottomIndex, bottom, connections, layerDict)
LayerHelper.__removeBlobPhaseFromDict(topPhase, bottom, blobOrder)
@staticmethod
def __addConnection(topLayerID, bottomLayerID, bottomBlobIndex, blobName, connections, layerDict):
topBlobIndex = LayerHelper.getTopBlobIndex(layerDict[topLayerID], blobName)
connections.append((topLayerID, topBlobIndex, bottomLayerID, bottomBlobIndex))
return
@staticmethod
def __removeBlobPhaseFromDict(phase, top, blobOrder):
blobList = blobOrder[top]
if len(blobList) > 1:
phaseDict = blobList[0]
if "" in phaseDict:
del phaseDict[""]
elif phase in blobList[1]:
del phaseDict[phase]
if len(phaseDict) == 0:
del blobList[0]
class OrderedLayersByBlob:
def __init__(self, layerDict, layerOrder):
self.blobLayerDict = dict()
self.__createOrderedLayerListPerBlob(layerDict, layerOrder)
def __createOrderedLayerListPerBlob(self, layerDict, layerOrder):
for layerID in layerOrder:
layer = layerDict[layerID]
self.__addLayerToDict(layer, layerID)
def __addLayerToDict(self, layer, layerID):
if "top" in layer["parameters"]:
for top in layer["parameters"]["top"]:
self.__addTopBlobLayer(top, layer, layerID)
def __addTopBlobLayer(self, top, layer, layerID):
if top in self.blobLayerDict:
if LayerHelper.isLayerInPlaceForBlob(layer, top):
if len(self.blobLayerDict[top]) > 1:
currentLevelDict = self.blobLayerDict[top][-1]
if "" in currentLevelDict or LayerHelper.getLayerPhase(layer) in currentLevelDict:
self.__createDictForTop(top, layer, layerID)
else:
currentLevelDict[LayerHelper.getLayerPhase(layer)] = layerID
else:
self.__createDictForTop(top, layer, layerID)
else:
self.blobLayerDict[top][0][LayerHelper.getLayerPhase(layer)] = layerID
else:
self.blobLayerDict[top] = list()
self.__createDictForTop(top, layer, layerID)
def __createDictForTop(self, top, layer, layerID):
tempDict = dict()
tempDict[LayerHelper.getLayerPhase(layer)] = layerID
self.blobLayerDict[top].append(tempDict)
Barista/gui/network_manager/network_manager.py 0000664 ڊ0011117 00000154047 13265130053 022103 0 ustar klemms o0cvpr import copy
import re
import sys
import uuid
from PyQt5.QtCore import QObject, pyqtSignal, Qt
from PyQt5.QtWidgets import QMessageBox
from gui.main_window.docks.properties.property_widgets.network_dict_info import NetworkDictInfoBuilder
from gui.main_window.docks.properties.property_widgets.program_state_info import ProgramStateInfoBuilder, defaultState
from gui.network_manager.history_manager import HistoryManager
import gui.main_window.docks.properties.property_widgets.data as PropData
from backend.barista.constraints.permanent.solver import ensureSolverConstraints
from backend.barista.utils.logger import Log
from backend.caffe import loader, saver, proto_info, dict_helper as helper
from gui.main_window.docks.properties.property_widgets.data import ReasonUpdated, ReasonKeyDeleted, ReasonKeyAdded, ReasonRemoved
from gui.network_manager.layer_helper import LayerHelper
def buildNetData(network):
""" Build PropertyData for the network-dictionary network """
netinfo = NetworkDictInfoBuilder(network)
return PropData.PropertyData(network, netinfo.buildRootInfo().subparameter(), netinfo)
def buildStateData(stateDict):
""" Build PropertyData for the state-dictionary stateDict """
if stateDict == None:
stateDict = defaultState()
info = ProgramStateInfoBuilder(stateDict)
return PropData.PropertyData(stateDict, info.buildRoot().subparameter(), info, signalcontroller=PropData.SignalController())
class HistoryWriting(object):
""" Class which should give methods to avoid recursive history writings """
def __init__(self, state, callback):
""" Creates this object.
state is the state-object which can be changed
through the makeEntry-call
callback is a function which will be called
if an entry in history should be make
"""
self.__state = state
self.__callback = callback
self.__lock = False
def valueOf(self,uri):
return self.__state.valueOf(uri)
def makeEntry(self, func):
""" Make an entry in history.
func is a function which will be called
with the state object
(e.g. historywriter.makeEntry(lambda state: state["name"] = 2)
as long as func is called, no other function can make an entry in history,
but they still can change the state through this function
"""
prevLocked = self.__lock
self.__lock = True
res = None
if func:
res = func(self.__state)
if not prevLocked:
self.__lock = False
self.__callback()
return res
def entryWithoutHistory(self,func):
""" Change state without touching the history. See makeEntry for the argument """
prevLocked = self.__lock
self.__lock = True
res = None
if func:
res = func(self.__state)
if not prevLocked:
self.__lock = False
return res
class NetworkManager(QObject):
""" stateChanged is triggered if the state of the network or solver has changed.
modifiedChanged is used to indicate a change of the network to other components.
The boolean value indicates whether the current state differs from the last saved one.
"""
stateChanged = pyqtSignal()
modifiedChanged = pyqtSignal(bool)
newStateData = pyqtSignal(object)
'The network manager stores the information for all layers currently in ui and handles all layer-related events'
def __init__(self, dockElementActiveLayers, dockLayerProperties, dockSolverProperties, nodeEditor):
'Creates a new network manager'
super(NetworkManager, self).__init__()
self.callerId = Log.getCallerId('networkmanager')
# Set the reference to the active layers dock, as well as reference to network manager in active layers dock
self.dockElementActiveLayers = dockElementActiveLayers.getLayersListWidget()
self.dockElementActiveLayers.setNetworkManager(self)
# Set the reference to property dock, as well as reference to network manager in property dock
self.dockLayerProperties = dockLayerProperties
self.dockLayerProperties.setNetworkManager(self)
# Set the reference to the solver property dock
self.dockSolverProperties = dockSolverProperties
#self.dockSolverProperties().solverChanged.connect(self.makeHistory)
self.dockSolverProperties().solverChanged.connect(self.validateChangedSolver, Qt.QueuedConnection)
# Set the reference to the node edtior scene, as well as reference to network manager in node prototxt_editor
self.nodeEditor = nodeEditor
self.nodeEditor.setNetworkManager(self)
# Get the caffe metainformation
self.caffeInfo = proto_info.CaffeMetaInformation()
# Set up a counter for overall number of layers in the application, to use as id for each layer
self.layerCount = 0
# Create an empty network, to be filled by both opening a file and user actions
self.stateData = None
self.historyWriter = None # type: HistoryWriting
self.setState(None)
# self.setNetwork(helper.bareNet("Unnamed"))
# Set up a list that contains all currently selected layer ids
# Setup historyManager
self.__historyDepth = 0
self._historymanager = HistoryManager()
self.makeHistory()
self.__lastModified = 0
self.stateChanged.connect(lambda: self.modifiedChanged.emit(self.isModified()))
self.networkHelper = None # type: helper.DictHelper
def validateChangedSolver(self):
"""This method is triggered automatically whenever the solver has been changed (in the GUI)."""
# prevent manipulation of solver constraints (changes self.solver)
data = self.dockSolverProperties().data()
if data is None:
data = self.solver
ensureSolverConstraints(data)
# ensureSolverConstraints(self.solver)
# backpropagate changed changes to the GUI by replacing the complete solver object
# (it's already up to date inside of this class, but not in the GUI)
# self.dockSolverProperties().setSolver(self.solver)
@property
def network(self):
return self.stateData["network"]
@property
def solver(self):
return self.stateData["solver"]
"""Method that adds a Layer in to the scene, called when drag and dropping a layer from
layer selection"""
def addLayer(self, type, scenePosX, scenePosY):
# Check if a layer have been moved
self.checkLayersForMovement(self.nodeEditor.getNodeIds())
""" Add a layer from the type (a string) at the given position """
def intern(stateData):
# generate new Layer Name
state = self.stateData.getBoringDict()
boringNetwork = state["network"]
nethelper = helper.DictHelper(boringNetwork)
# get all present Layer Names in a list
allLayerNames = nethelper.nameOfLayers()
# set new Name as Typename and layercount = 0
newName = type
count = 0
for name in allLayerNames:
if newName == re.sub(' #[0-9]+$', '', name):
try:
number = int(re.sub(newName + ' #', '', name))
count = number if number > count else count
except ValueError:
count = 0
# increment layertype count
count += 1
# append layertypecount to new Layername
newName = newName + " #" + str(count)
network = stateData["network"]
networkHelper = helper.DictHelper(network)
# Add a layer to the network
#self.stateData.signalController().lock()
newlayer, id = networkHelper.addLayer(self.caffeInfo.availableLayerTypes()[type], newName, 0)
#self.stateData.signalController().unlock()
# Add the node item to the graphics scene
stateData["position"][id] = [scenePosX, scenePosY]
#self.nodeEditor.addLayer(id, network['layers'][id], scenePosX, scenePosY)
# Get the order and add the layer to the dock of active layers
order = network["layerOrder"].index(id)
#self.dockElementActiveLayers.addItem(id, name, order, type)
self.updateOrder()
# Set the selection to the new element
self.setSelection(id)
# Set the new overall layer count
self.layerCount += 1
self.historyWriter.makeEntry(intern)
def focusLayer(self, id):
self.checkLayersForMovement(id)
self.nodeEditor.focusLayer(id)
def deleteLayer(self, id):
# Check if a layer have been moved
self.checkLayersForMovement(self.nodeEditor.getNodeIds())
""" Removes the layer with the given id """
def intern(stateDict):
self.setSelection(id)
self.deleteSelectedLayers()
self.historyWriter.makeEntry(intern)
def deleteSelectedLayers(self):
# Check if a layer have been moved
self.checkLayersForMovement(self.nodeEditor.getNodeIds())
'Deletes all Layers currently selected'
def intern(stateData):
stateData.signalController().lock()
self.nodeEditor.disconnectConnectionsOfSelection()
network = stateData["network"]
networkHelper = helper.DictHelper(network)
selected = list(stateData["selection"])
stateData["selection"].setValue([])
for id in selected:
self.__checkLayerForMovement(id)
if networkHelper.hasLayerWithId(id):
networkHelper.removeLayer(id)
del stateData["position"][id]
stateData.signalController().unlock()
self.updateOrder()
self.historyWriter.makeEntry(intern)
def onStateUpdate(self, uri, reason):
""" Get called if something is changed in stateData """
def intern(stateData):
# # Debugging output:
# stateData.signalController().enableLoging()
# print("Updating {} because {}".format(uri, reason))
# If something in selection changed
if uri[0] == "selection":
# If we have an complete new list of selected docks
if isinstance(reason, ReasonUpdated):
ids = reason.oldValue
# Trigger if layers moved
self.__checkLayerForMovement(ids)
# If a node get deselected
if isinstance(reason, ReasonRemoved):
ids = [id for id in stateData["selection"]]
ids.append(reason.value)
# Trigger if layers moved
self.__checkLayerForMovement(ids)
# Set the selection in prototxt_editor and the propertywdg
ids = [id for id in stateData["selection"]]
self.nodeEditor.setSelectedLayers(ids)
self.dockLayerProperties.setTabs(ids)
# The whole network changed
elif uri == ["network"]:
# Refresh node-items and connections
self.refreshItems()
self.refreshConnections()
elif uri == ["hidden_connections"]:
self.updateHiddenConnectionsInNodeEditor()
# Some layer information changed
elif uri[:2] == ["network","layers"] and len(uri) > 2:
id = uri[2]
# Some parameter changed
if len(uri) > 4 and uri[3] == "parameters":
self.updateToolTip(id)
reluri = uri[4:]
# The top blobs changed -> tell the node prototxt_editor
if reluri == ["top"]:
# Some blob added
if isinstance(reason, PropData.ReasonAdded):
idx = reason.idx
content = self.historyWriter.valueOf(uri+[idx])
self.nodeEditor.addTopBlob(id, content)
# Some blob removed
elif isinstance(reason, PropData.ReasonRemoved):
idx = reason.idx
self.nodeEditor.disconnectTopBlob(id, idx)
self.nodeEditor.removeTopBlob(id, idx)
self.updateOrder()
# The bottom blobs changed -> tell the node prototxt_editor
elif reluri == ["bottom"]:
# Some blob added
if isinstance(reason, PropData.ReasonAdded):
idx = reason.idx
content = self.historyWriter.valueOf(uri+[idx])
self.nodeEditor.addBottomBlob(id, idx)
# Some blob removed
elif isinstance(reason, PropData.ReasonRemoved):
idx = reason.idx
self.nodeEditor.disconnectBottomBlob(id, idx)
self.nodeEditor.removeBottomBlob(id, idx)
self.updateOrder()
# One top blobs changed -> tell the node prototxt_editor
elif reluri[0] == "top" and isinstance(reason, PropData.ReasonUpdated):
newVal = reason.newValue
self.nodeEditor.renameTopBlob(id, reluri[1], newVal)
# One bottom blobs changed -> tell the node prototxt_editor
elif reluri[0] == "bottom" and isinstance(reason, PropData.ReasonUpdated):
newVal = reason.newValue
self.nodeEditor.renameBottomBlob(id, reluri[1], newVal)
elif reluri[0] == "name" and isinstance(reason, PropData.ReasonUpdated):
if reason.oldValue != reason.newValue:
self.nodeEditor.updateLayerData(id)
layerNames = [self.nodeEditor.getNodes()[currentId].getName() for currentId in self.nodeEditor.getNodes().keys()]
layerNames.remove(reason.newValue)
if reason.newValue in layerNames:
buttonReply = QMessageBox.question(None, "Warning!",
"A layer with that name already exists. "
"During training, caffe will assume shared weights among equally named layers. "
"Training will fail if dimensions don't match.\n\n"
"Do you still want to change the name?",
QMessageBox.Yes | QMessageBox.No)
if buttonReply == QMessageBox.No:
self.undo()
else:
pass
# The include phase changed -> tell the node prototxt_editor
else:
self.nodeEditor.updateLayerData(id)
# Layers are changed
elif uri == ["network", "layers"]:
# If a layer is removed -> tell the node prototxt_editor
if isinstance(reason, ReasonKeyDeleted):
# Check if some node has moved
id = reason.key
layerstocheck = list(stateData["selection"])
if id in layerstocheck:
layerstocheck.remove(id)
self.__checkLayerForMovement(layerstocheck)
self.nodeEditor.deleteItemsByID([id])
self.updateOrder()
# If a layer is added -> tell the node prototxt_editor
if isinstance(reason,ReasonKeyAdded):
# Check if some node has moved
self.__checkLayerForMovement(list(stateData["selection"]))
id = reason.key
self.nodeEditor.addLayer(id, stateData["network"]["layers"][id], 0.0,0.0)
# Positions get updated -> tell the node prototxt_editor
elif uri[0] == "position" and (isinstance(reason, ReasonUpdated) or isinstance(reason,ReasonKeyAdded)):
self.nodeEditor.applyLayerPositionDict(stateData["position"])
# We've got a new solver -> set solver to the propertywdg
elif uri == ["solver"]:
self.setSolver(stateData["solver"])
#stateData.signalController().disableLoging()
self.historyWriter.entryWithoutHistory(intern)
self.newStateData.emit(self.getStateDictionary())
def setSelection(self, id):
""" Select the layer with the given id.
Note: you can also manipulate stateData["selection"] instead.
"""
self.setSelectionList([id])
def setSelectionList(self, ids):
""" Select all layers in ids
Note: you can also manipulate stateData["selection"] instead.
"""
def intern(stateData):
stateData["selection"].setValue(ids)
self.historyWriter.makeEntry(intern)
def addLayerToSelection(self, id):
""" Add the layer with the given id to the selection.
Note: you can also manipulate stateData["selection"] instead.
"""
self.historyWriter.makeEntry(lambda stateData: stateData["selection"].pushBack(id))
# Notify all elements
#self.dockElementActiveLayers.addItemToSelection(id)
#self.nodeEditor.addLayerToSelection(id)
#self.dockLayerProperties.addTab(id)
# Save the history to include this step in undo and redo
#self.__historyDepth -= 1
#self.makeHistory()
def __checkLayerForMovement(self,ids):
""" Checks if the layer with ids (a list of string or a string)
has a different position in prototxt_editor than in stateData.
If so the stateData will be updated.
"""
if type(ids) != list:
ids = [ids]
def intern(stateData):
# mayUpdate contains ids which are valid
# because some if ids could already be deleted
mayUpdate = set(ids).intersection(stateData["position"].keys())
oldValues = dict([(id, stateData["position"][id]) for id in mayUpdate])
newValue = dict([(id,self.nodeEditor.getPositionOfLayer(id)) for id in mayUpdate])
for id in mayUpdate:
oldx, oldy = oldValues[id]
newx, newy = newValue[id]
if oldx != newx or oldy != newy:
stateData["position"][id].setValue([newx,newy])
self.historyWriter.makeEntry(intern)
def checkLayersForMovement(self,ids):
self.__checkLayerForMovement(ids)
def removeLayerFromSelection(self, id):
""" Deselect the layer with the given id.
Note: you can also manipulate stateData["selection"] instead.
"""
def intern(stateData):
stateData["selection"].remove(id)
self.historyWriter.makeEntry(intern)
def clearSelection(self):
""" Deselect all layers.
Note: you can also manipulate stateData["selection"] instead.
"""
if len(self.stateData["selection"]) == 0: return
def intern(stateData):
stateData["selection"].setValue([])
self.historyWriter.makeEntry(intern)
def clearSelectionWithoutSavingHistory(self):
""" Deselect all layers without an entry in history.
Note: you can also manipulate stateData["selection"]
through historyWriter.entryWithoutHistory instead.
"""
self.historyWriter.entryWithoutHistory(lambda state: self.clearSelection())
def updateNet(self):
""" Force an entry in history """
self._historymanager.insertFunc(None)
# todo: this should really do something
def updateName(self, id, name):
""" Sets the name of the layer with the given id
to the given name.
Note: you can also manipulate
stateData["network"]["layers"][id]["parameters"]["name"]
instead
"""
def intern(stateDict):
network = stateDict["network"]
networkHelper = helper.DictHelper(network)
networkHelper.layerParams(id)["name"] = name
print("updateName: ", id, name)
self.historyWriter.makeEntry(intern)
def updateLayerInNodeEditor(self, layerID):
""" Tell nodeEditor to update the layerdata of the given layer """
self.nodeEditor.updateLayerData(layerID)
def updateOrder(self):
""" Recalculate the order of the layer """
if set(self.nodeEditor.getNodeIds()) == set(self.network["layers"].keys()):
self.network["layerOrder"].setValue(self.nodeEditor.calculateLayerOrder())
def refreshConnections(self):
""" Refresh the connections of all layers """
# Connect layers
connections = LayerHelper.buildConnectionList(self.network.getBoringDict())
#connections = LayerHelper.buildConnectionList(self.network)
for connection in connections:
self.nodeEditor.createConnection(connection[0], connection[1], connection[2], connection[3])
def updateHiddenConnectionsInNodeEditor(self):
""" Update the NodeEditor to hide connections which
should be hidden in the current state
"""
# There are no hidden_connections
if not "hidden_connections" in self.stateData:
self.nodeEditor.setHiddenConnectionsFromList([])
return
connectionList = []
for el in self.stateData["hidden_connections"]:
topId = el["topLayerId"]
topIdx = el["topBlobIdx"]
bottomId = el["bottomLayerId"]
bottomIdx = el["bottomBlobIdx"]
connectionList.append(( topId, topIdx, bottomId, bottomIdx ))
self.nodeEditor.setHiddenConnectionsFromList(connectionList)
def connectionsHiddenStateChanged(self, listOfConnections):
""" Gets called when a connection gets hidden/shown. listOfConnections is a list of tupels (topLayerID, topBlobIndex, bottomLayerID, bottomBlobIndex) """
hidden_connections = []
for (topid, topidx, bottomid, bottomidx) in listOfConnections:
hidden_connections.append({
"topLayerId": topid,
"topBlobIdx": topidx,
"bottomLayerId": bottomid,
"bottomBlobIdx": bottomidx
})
self.historyWriter.makeEntry(lambda stateData: stateData.giveProperty("hidden_connections", hidden_connections))
def refreshItems(self):
# Clear everything, if a previous network has been opened
self.nodeEditor.clearAll()
self.dockLayerProperties.clearProperties()
self.dockElementActiveLayers.clearTable()
for id, layer in self.getLayerDict().iteritems():
# todo: Set an ID for each layer in input dictionary
# Create a new node item in the view and set focus to it
self.nodeEditor.addLayer(id, layer)
# Create a new entry in the active layers list
self.dockElementActiveLayers.addItem(id,
layer["parameters"]["name"],
self.network["layerOrder"].index(id),
layer["parameters"]["type"])
# Add one to the count of all layers in the application
self.layerCount += 1
def addBottomBlob(self, layerID, blobName=""):
'Adds a bottom entry to the layer with the given id'
def intern(stateDict):
network = stateDict["network"]
networkHelper = helper.DictHelper(network)
layerParams = networkHelper.layerParams(layerID)
if "bottom" not in layerParams:
layerParams.giveProperty("bottom",[])
layerParams["bottom"].pushBack(blobName)
# TODO: Instead update in onStateUpdate
#self.nodeEditor.addBottomBlob(layerID, "")
# update property dock
self.dockLayerProperties.updateDock()
self.historyWriter.makeEntry(intern)
def removeBottomBlob(self, layerID, blobIndex):
def intern(stateDict):
network = stateDict["network"]
networkHelper = helper.DictHelper(network)
self.nodeEditor.disconnectBottomBlob(layerID, blobIndex)
del networkHelper.layerParameter(layerID, "bottom")[blobIndex]
# TODO: Instead update in onStateUpdate
# self.nodeEditor.removeBottomBlob(layerID, blobIndex)
self.updateOrder()
# update property dock
self.dockLayerProperties.updateDock()
self.historyWriter.makeEntry(intern)
def addTopBlob(self, layerID, blobName):
'Adds a top entry to the layer with the given id'
def intern(stateDict):
network = stateDict["network"]
networkHelper = helper.DictHelper(network)
layerParams = networkHelper.layerParams(layerID)
if "top" not in layerParams:
layerParams.giveProperty("top", [])
layerParams["top"].pushBack(blobName)
# TODO: Instead update in onStateUpdate
#self.nodeEditor.addTopBlob(layerID, blobName)
# update property dock
self.dockLayerProperties.updateDock()
self.historyWriter.makeEntry(intern)
def renameTopBlob(self, layerID, blobIndex, newName):
def intern(stateDict):
network = stateDict["network"]
networkHelper = helper.DictHelper(network)
networkHelper.layerParams(layerID)["top"][blobIndex] = newName
# TODO: Instead update in onStateUpdate
#self.nodeEditor.renameTopBlob(layerID, blobIndex, newName)
# update property dock
self.dockLayerProperties.updateDock()
self.historyWriter.makeEntry(intern)
def renameBottomBlob(self, layerID, blobIndex, newName):
def intern(stateDict):
network = stateDict["network"]
networkHelper = helper.DictHelper(network)
networkHelper.layerParams(layerID)["bottom"][blobIndex] = newName
# TODO: Instead update in onStateUpdate
#self.nodeEditor.renameBottomBlob(layerID, blobIndex, newName)
# update property dock
self.dockLayerProperties.updateDock()
self.historyWriter.makeEntry(intern)
def removeTopBlob(self, layerID, blobIndex):
def intern(stateDict):
network = stateDict["network"]
networkHelper = helper.DictHelper(network)
self.nodeEditor.disconnectTopBlob(layerID, blobIndex)
del networkHelper.layerParams(layerID)["top"][blobIndex]
# TODO: Instead update in onStateUpdate
#self.nodeEditor.removeTopBlob(layerID, blobIndex)
self.updateOrder()
# update property dock
self.dockLayerProperties.updateDock()
self.historyWriter.makeEntry(intern)
def connectLayers(self, topLayerID, topBlobIndex, bottomLayerID, bottomBlobIndex):
def intern(stateDict):
network = stateDict["network"]
networkHelper = helper.DictHelper(network)
blobName = networkHelper.layerParams(topLayerID)["top"][topBlobIndex]
networkHelper.layerParams(bottomLayerID)["bottom"][bottomBlobIndex] = blobName
# TODO: Instead update in onStateUpdate
self.nodeEditor.updateLayerData(bottomLayerID)
self.nodeEditor.createConnection(topLayerID, topBlobIndex, bottomLayerID, bottomBlobIndex)
# self.updateNet()
self.updateOrder()
# update property dock
self.dockLayerProperties.updateDock()
self.historyWriter.makeEntry(intern)
def disconnectLayer(self, bottomLayerID, bottomBlobIndex):
def intern(stateDict):
network = stateDict["network"]
networkHelper = helper.DictHelper(network)
networkHelper.layerParams(bottomLayerID)["bottom"][bottomBlobIndex] = unicode("")
# TODO: Instead update in onStateUpdate
self.nodeEditor.updateLayerData(bottomLayerID)
#self.updateNet()
self.updateOrder()
# update property dock
self.dockLayerProperties.updateDock()
self.historyWriter.makeEntry(intern)
def isValidLayerType(self, type):
if type in self.caffeInfo.availableLayerTypes():
return True
return False
def __historyEntry(self, uri, reason):
self.historyWriter.makeEntry(None)
def setState(self, statedict):
""" Sets a new stateData and refresh the gui.
statedict has to be from type dict and
will be wrapped with PropertyData through this function.
"""
# Disconnect old signals
if self.stateData:
self.stateData.someChildPropertyChanged.disconnect(self.onStateUpdate)
self.stateData.someChildPropertyChanged.disconnect(self.__historyEntry)
# Build PropertyData
self.stateData = buildStateData(statedict)
# Refresh everynthing
self.__connectReactiveThings()
self.refreshItems()
self.refreshConnections()
self.nodeEditor.applyLayerPositionDict(self.stateData["position"])
self.nodeEditor.setSelectedLayers(self.stateData["selection"])
self.updateHiddenConnectionsInNodeEditor()
if self.stateData:
self.historyWriter = HistoryWriting(self.stateData, self.makeHistory)
def intern(stateData):
self.networkHelper = helper.DictHelper(stateData["network"])
self.dockLayerProperties.setTabs(stateData["selection"])
# Reconnect
self.stateData.someChildPropertyChanged.connect(self.__historyEntry)
self.stateData.someChildPropertyChanged.connect(self.onStateUpdate)
if "solver" in self.stateData:
self.setSolver(stateData["solver"])
else:
self.setSolver(None)
self.historyWriter.entryWithoutHistory(intern)
def makeHistory(self):
""" Make an entry in history.
Note: If possible use historyWriter.makeEntry instead
as it avoid making unnecessary entries.
"""
#if self.__historyDepth > 0:
# return
boring = self.stateData.getBoringDict()
network = self.stateData["network"].getBoringDict()
valid, msg = helper.isNetDictValid(network)
if not valid:
Log.error(
"WARNING!!!! Network-Dictionary is not valid:"
+" " + msg
+"\nInfo: Current Dictionary looks like this:\n"
+ str(self.network.getBoringDict()),
self.callerId
)
self._historymanager.insertFunc(lambda insert: insert(copy.deepcopy(self.stateData.getBoringDict())))
netids = set(network["layerOrder"])
posids = set(self.stateData["position"].getBoringDict())
if netids != posids:
Log.error("WARNING!!! NetworkManager has invalid History-State",self.callerId)
for id in netids.difference(posids):
Log.error("Id {} is in net, but has no position".format(id),self.callerId)
for id in posids.difference(netids):
Log.error("Id {} has position, but is not in net".format(id),self.callerId)
Log.log("Info: Current State-Dictionary looks like this:",self.callerId)
Log.log(str(self._historymanager.currentState()),self.callerId)
self.stateChanged.emit()
def _refreshStateFromHistory(self):
""" Rebuild State of network, selection, position from latest point in history """
state = copy.deepcopy(self._historymanager.currentState())
def updateFun():
self.setState(state)
#self.setNetwork(state["network"])
#self.refreshItems()
#self.refreshConnections()
#self.setSelectionList(state["selection"])
#self.setLayerPositionDictionary(state["position"])
#self.setSolver(state["solver"])
#if not state is None:
self._historymanager.lockHistory(updateFun)
self.stateChanged.emit()
def canUndo(self):
return self._historymanager.canUndo()
def undo(self):
self._historymanager.undo()
self._refreshStateFromHistory()
def canRedo(self):
return self._historymanager.canRedo()
def redo(self):
self._historymanager.redo()
self._refreshStateFromHistory()
def isModified(self):
""" Return True iff anything changes since last reset of changes """
return self.__lastModified != self._historymanager.position
def resetModifiedFlag(self):
""" Reset the change flag, so the program things nothing has changes
since last save.
Then there is no need to ask the user for example.
"""
self.__lastModified = self._historymanager.position
self.modifiedChanged.emit(self.isModified())
def setModifiedFlag(self):
""" Set the change flag, so the program things something has changes
since last save.
So there is need to ask the user for example.
"""
self.__lastModified = -1
self.modifiedChanged.emit(self.isModified())
def openNetworkFromFile(self, filePath):
with open(filePath, 'r') as file:
inputString = file.read()
self.openNetworkFromString(inputString)
def saveNetworkToFile(self, filePath):
with open(filePath, 'w') as file:
file.write(saver.saveNet(self.network))
def openNetworkFromString(self, inputString, clearHistory = True):
if clearHistory:
self._historymanager.clear()
def intern(stateData):
self.setNetwork(loader.loadNet(inputString))
#self.refreshItems()
#self.refreshConnections()
# Rearrange the layers
self.nodeEditor.rearrangeNodes()
self.updateOrder()
self.historyWriter.makeEntry(intern)
def checkForDuplicateNames(self):
"""
Check for duplicate layer names and send a warning but not if they have different phases.
"""
state = self.stateData.getBoringDict()
network = state["network"]
nethelper = helper.DictHelper(network)
if nethelper:
layerNames = nethelper.nameOfLayers()
layerNamesSet = set(layerNames)
if not len(layerNames) == len(layerNamesSet):
# Extract only duplicate layer names
duplicateNames = set([name for name in layerNames if not name in layerNamesSet or layerNamesSet.remove(name)])
for name in duplicateNames:
phases = []
noPhase = False
for id in nethelper.layerIdsForName(name):
params = nethelper.layerParams(id)
if params and "include" in params and len(params["include"]) > 0 and "phase" in params["include"][0]:
phases.append(params["include"][0]["phase"])
else:
noPhase = True
break
if noPhase or not len(phases) == len(set(phases)):
QMessageBox.about(None, "Warning!", "A layer with that name already exists. "
"During training, caffe will assume shared weights among equally named layers. "
"Training will fail if dimensions don't match.")
return
def setNetwork(self, network):
def intern(stateData):
stateData.signalController().lock()
stateData.giveProperty("network", network)
stateData["position"].setValue(dict([(id, [0.0,0.0]) for id in network["layerOrder"]]))
stateData["selection"].setValue([])
stateData.signalController().unlock()
self.historyWriter.entryWithoutHistory(intern)
self.networkHelper = helper.DictHelper(self.stateData["network"])
def __connectReactiveThings(self):
if self.dockElementActiveLayers:
self.dockElementActiveLayers.setReactiveState(self.stateData)
def openSolverFromFile(self, filePath):
with open(filePath, 'r') as file:
inputString = file.read()
self.openSolverFromString(inputString)
def openSolverFromString(self,inputString, clearHistory=False):
if clearHistory:
self._historymanager.clear()
def intern(stateDict):
stateDict.giveProperty("solver",loader.loadSolver(inputString))
self.historyWriter.makeEntry(intern)
def setSolver(self, solver):
# prevent manipulation of solver constraints
if solver:
solver = ensureSolverConstraints(solver)
# save data
self.dockSolverProperties().setSolver(solver)
self.historyWriter.makeEntry(None)
def setLayerPositionDictionary(self, posDict):
self.nodeEditor.applyLayerPositionDict(posDict)
def getLayerPositionDictionary(self):
return self.nodeEditor.createLayerPositionDict()
def getLayerDict(self):
return self.network["layers"]
def getLayer(self, layerId):
return self.networkHelper.layerPerId(layerId)
def getLayerCount(self):
return self.layerCount
def getLayerTypeCount(self, type):
return self.dicLayerTypeCounter[type]
def setStateDictionary(self, dictionary, clearHistory = False):
""" Sets the dictionary of docks state and update gui """
if clearHistory:
self._historymanager.clear()
self._historymanager.insertFunc(lambda inserter: inserter(dictionary))
self._refreshStateFromHistory()
def getStateDictionary(self):
""" Returns a dictionary which represents the current state of the project """
#return self._historymanager.currentState()
return self.stateData.getBoringDict()
def getToolTip(self, id):
'Returns the standard tooltip for an item representing the layer given by id'
# Check if name exists, since it is optional
network = self.stateData["network"].getBoringDict()
if "name" in network["layers"][id]["parameters"]:
tooltip = network["layers"][id]["parameters"]["name"] + "\n"
# Add type and other parameters
tooltip += "Type: " + network["layers"][id]["parameters"]["type"] + "\n"
tooltip += "\n"
tooltip += "Parameters: " + "\n"
# Add all other parameters to the tooltip
for key in network["layers"][id]["parameters"]:
if not key in ["name", "type", "top", "bottom"]:
tooltip += self._getToolTipHelper(0, key, network["layers"][id]["parameters"][key])
tooltip += "\n"
# Add Top and Bottom to the bottom of the tooltip
if "top" in network["layers"][id]["parameters"]:
tooltip += self._getToolTipHelper(0, "top", network["layers"][id]["parameters"]["top"])
if "bottom" in network["layers"][id]["parameters"]:
tooltip += self._getToolTipHelper(0, "bottom", network["layers"][id]["parameters"]["bottom"])
return tooltip
def _getToolTipHelper(self, tabs, key, data):
# Initialize the partial tooltip as a string containing the given key
toolTipPart = tabs * " "
toolTipPart += key
# Use recursion if the data is a dict
if isinstance(data, dict):
toolTipPart += " { \n"
for innerKey in data:
# Add a tabulator as starter string for inner parts of the dict
toolTipPart += self._getToolTipHelper(tabs + 1, innerKey, data[innerKey])
toolTipPart += tabs * " " + "} \n"
elif isinstance(data, list):
# Set up a special case for the representation of top and bottom lists
if (key == "top") | (key == "bottom"):
toolTipPart += ": "
for i in range(0, len(data)):
# The string may contain some spacial characters which can not be displayed in unicode.
# The encode method returns a bytes representation of the unicode string and ignores errors.
# Then the decode method convertes the string using the utf-8 codec.
# This string can be displayed even if it is containing special characters.
toolTipPart += "\t" + data[i].encode('utf-8', 'ignore').decode('utf-8') + "\n"
else:
# Proceed based on the type of the list elements, catch empty lists
if len(data) > 0:
# Assumption: All elements of the parameter list are of the same type
# always true in current network definitions. If that is changed, this code must be revised
if isinstance(data[0], int) | isinstance(data[0], float):
# In case of simple int or float values, just concatenate the elements into one line of string
toolTipPart += ": "
for i in range(0, len(data)):
toolTipPart += str(data[i]) + " "
toolTipPart += "\n"
elif isinstance(data[0], dict):
# In case of more complex element types, create a new subset of values containing the elements
toolTipPart += " [ \n"
# Type is dictionary, use recursion to get all values from that dicationary
for i in range(0, len(data)):
for key in data[i]:
toolTipPart += self._getToolTipHelper(tabs + 1, key, data[i][key])
toolTipPart += tabs * " " + "] \n"
elif isinstance(data[0], list):
# In case of more complex element types, create a new subset of values containing the elements
toolTipPart += " [ \n"
# Type is list, user recursion to get all values from that list
for i in range(0, len(data)):
toolTipPart += self._getToolTipHelper(tabs, "", data[i])
toolTipPart += tabs * " " + "] \n"
elif isinstance(data, str) or isinstance(data, unicode):
# Add string by simple concatenation
toolTipPart += ": " + "\"" + data + "\"" + "\n"
elif isinstance(data, int) or isinstance(data, float):
# Add int by simple concatenation
toolTipPart += ": " + str(data) + "\n"
return toolTipPart
def updateToolTip(self, layerID):
self.nodeEditor.updateTooltip(layerID)
def duplicateLayers(self, layerIDs, point=None):
""" Duplicates the layer referenced by layerIDs. If these are interconnected, then the connection is duplicated
as well.
"""
# Get the current data
state = self.stateData.getBoringDict()
network = state["network"]
nethelper = helper.DictHelper(network)
# Check if a layer have been moved
self.checkLayersForMovement(self.nodeEditor.getNodeIds())
layerIDs = self._checkValidIDs(layerIDs, nethelper)
newLayerNames = self._generateNewLayerNames(layerIDs, nethelper)
newIDs = self._generateNewLayers(layerIDs, newLayerNames, nethelper, state)
self._addBlobs(layerIDs, newIDs, newLayerNames, nethelper)
self._addConnections(layerIDs, newIDs)
self._translateLayers(layerIDs, newIDs, point)
def _checkValidIDs(self, layerIDs, nethelper):
""" check if every id is valid, returns a list of all valid IDs in layerIDs. """
tempLayerIds = []
for index, id in enumerate(layerIDs):
# encapsulate getting the layer per id in a try block
try:
nethelper.layerPerId(id)
tempLayerIds.append(id)
except KeyError:
Log.error(
"Could not copy element with id " + str(id),
self.callerId
)
return tempLayerIds
def _generateNewLayerNames(self, layerIDs, nethelper):
""" This function generates new Layer names based on a list of layerIds to be duplicated and the already given
names in the network, to prevent duplicate names or unreasonable enumeration.
The names are generated as followed:
1. Check if the layer to be copied ends in a number.
If it does:
add 1 to this number
If it does not:
append #1 at the end
2. Increase the last number until the string does not match any given layer name in the network or of the
new names already generated in the loop.
"""
# get all already distributed names, including the new generated names in the following loop
allLayerNames = nethelper.nameOfLayers()
# list to save the new generated Names
newNames = []
for id in layerIDs:
# Get the old layer and its name
oldLayer = nethelper.layerPerId(id)
oldName = oldLayer["parameters"]["name"]
# Check if oldName ends with a number, then strips the number
# baseName stored the name without the number at the end
baseName = re.sub('[0-9]+$', '', oldName)
# check if there was a number attached at the end, store it in counter
try:
counter = int(oldName[len(baseName):])
counter += 1
except ValueError:
# Set up new counter, add a '#' at the basename to indicate a new enumeration
counter = 1
baseName = baseName + "#"
newName = "{}{}".format(baseName, counter)
# Check if there already is a name with that number, if yes, add one to counter for new name
while newName in allLayerNames:
counter += 1
newName = "{}{}".format(baseName, counter)
# Append the chosen Name
allLayerNames.append(newName)
newNames.append(newName)
return newNames
def _generateNewLayers(self, layerIDs, newLayerNames, nethelper, state):
""" Given a list of given layerIDs, and a list of new layer names, duplicate the layers in the first list by
using the names of the latter. The i-th elements of the two lists correspond to one another.
Returns a list of the IDs of the new layers.
"""
# Get the current data
positions = state["position"]
network = state["network"]
newIDs = []
for index in range(len(layerIDs)):
id = layerIDs[index]
newName = newLayerNames[index]
# Get the old layer and its name
oldLayer = nethelper.layerPerId(id)
# Create a new layer with the type and parameters of the old layer
newLayer = {
"type": oldLayer["type"],
"parameters": copy.deepcopy(oldLayer["parameters"])
}
# Overwrite the values for name, top and bottom
newLayer["parameters"]["name"] = newName
newLayer["parameters"]["top"] = []
newLayer["parameters"]["bottom"] = []
# Generate random new ID and set the new idx
newId = str(uuid.uuid4())
idx = network["layerOrder"].index(id) + 1
# Add the new layer to the network
network["layers"][newId] = newLayer
network["layerOrder"].insert(idx, newId)
newIDs.append(newId)
# Set the new position for each copied element, also replace selection with duplicates
newPositions = positions.copy()
for i in range(0, len(layerIDs)):
(x, y) = positions[layerIDs[i]]
newPositions[newIDs[i]] = (x + 20, y + 20)
# Update the network manager
self.setStateDictionary({
"position": newPositions,
"selection": newIDs,
"network": network,
"solver": state["solver"]
})
return newIDs
def _addBlobs(self, layerIDs, newIDs, newLayerNames, nethelper):
""" Add bottom and uniquely named top blobs to the layers referenced by newIDs, by using the blobs in the layers
referenced by layerIDs as a template.
The naming of the top blobs goes as follows:
1. If the name of the blob in the old layer matches the name of the layer:
The new blob gets the name of the new layer (which always ends in a number).
Else:
If the old blob end with a number:
Add one to that number
Else:
Append "#1" to the name
2. Increase the last number until the string does not match any given layer name in the network or of the
new names already generated in the loop.
"""
allTopNames = nethelper.nameOfTopBlobs()
# Add the right top and bottom blobs to all new layers
for index in range(0, len(layerIDs)):
newLayerName = newLayerNames[index]
oldLayer = nethelper.layerPerId(layerIDs[index])
oldLayerName = oldLayer["parameters"]["name"]
newID = newIDs[index]
oldID = layerIDs[index]
# Create the right amount of bottom blobs for the new layer
if "bottom" in nethelper.layerPerId(oldID)["parameters"]:
for j in range(0, len(nethelper.layerPerId(oldID)["parameters"]["bottom"])):
self.addBottomBlob(newID)
# Create the right amount of top blobs for the new layer, using the names from the old one
if "top" in nethelper.layerPerId(oldID)["parameters"]:
for j in range(0, len(nethelper.layerPerId(oldID)["parameters"]["top"])):
topBlobName = nethelper.layerPerId(oldID)["parameters"]["top"][j]
# Check if top blob name matches the layername
if topBlobName == oldLayerName:
# check if newLayerName is candidate for top blob name
candidateName = newLayerName
if candidateName in allTopNames:
candidateName += "#1"
# Since newName ends with a number (either as an iteration based on a previous name or by adding
# #1), we can assert here that candidateName ends with a number
else:
candidateName = topBlobName
# Check if there is no number at the end of the top blob name
baseName = re.sub('[0-9]+$', '', candidateName)
if baseName >= candidateName:
candidateName += '#1'
# Set up the current counter, either by setting it to 1 or by getting the suffix, if it is a number
baseName = re.sub('[0-9]+$', '', candidateName)
try:
counter = int(candidateName[len(baseName):])
except ValueError:
# Set up new counter, add a '#' at the basename to indicate a new enumeration
counter = 1
baseName = baseName + "#"
newTopName = "{}{}".format(baseName, counter)
# increase pending number of candidateName, until it is unique.
while newTopName in allTopNames:
baseName = re.sub('[0-9]+$', '', newTopName)
counter += 1
newTopName = "{}{}".format(baseName, counter)
self.addTopBlob(newIDs[index], newTopName)
# Add new name to the list to prevent duplicates
allTopNames.append(newTopName)
def _addConnections(self, layerIDs, newIDs):
""" Check if copied layers were interconnected, if yes, add these connections to the new layers. """
for i in range(0, len(layerIDs)):
# Get the corresponding layer item
nodeItem = self.nodeEditor.getNodes()[layerIDs[i]]
for j in range(0, len(nodeItem.getBottomConnectors())):
for k in range(0, len(nodeItem.getBottomConnectors()[j].getConnections())):
for n in range(0, len(layerIDs)):
if nodeItem.getBottomConnectors()[j].getConnections()[k].getTopConnector() in \
self.nodeEditor.getNodes()[layerIDs[n]].getTopConnectors():
self.connectLayers(newIDs[n], nodeItem.getBottomConnectors()[j].getConnections()[
k].getTopConnector().getIndex(),
newIDs[i], nodeItem.getBottomConnectors()[j].getIndex())
def _translateLayers(self, layerIDs, newIDs, point):
""" If a position has been given by user, translate the new layers. """
if point != None:
# Find the smallest x coordinate from the old layers
minX = sys.maxint
nodes = self.nodeEditor.getNodes()
for id in layerIDs:
if nodes[id].x() < minX:
minX = nodes[id].x()
correspondingY = nodes[id].y()
# Translate all new elements accordingly
for i in range(0, len(newIDs)):
nodes[newIDs[i]].setPos(nodes[layerIDs[i]].x() - minX + point.x(),
nodes[layerIDs[i]].y() - correspondingY + point.y())
# todo: IMPORTANT how do i save these positions so that they won't get reversed?
Barista/gui/prototxt_editor/__init__.py 0000664 ڊ0011117 00000000000 13265130053 020520 0 ustar klemms o0cvpr Barista/gui/prototxt_editor/editor_search_bar.py 0000664 ڊ0011117 00000005711 13265130053 022436 0 ustar klemms o0cvpr from PyQt5.QtWidgets import QLineEdit, QPlainTextEdit
from PyQt5.QtCore import Qt
from PyQt5.QtGui import QTextCursor, QTextCharFormat, QTextDocument, QTextCursor, QColor, QFontMetrics
class SearchBar(QLineEdit):
def __init__(self, editor, parent=None):
QLineEdit.__init__(self, parent)
self.editor = editor
self.setAttribute(Qt.WA_TintedBackground)
self.setHidden(True)
self.defaultstyle = self.styleSheet()
self.textChanged.connect(self.onTextChange)
def disp(self):
'''display the searchbar'''
self.setHidden(False)
self.setFocus()
def hide(self):
'''hide the searchbar'''
self.setHidden(True)
def onF3(self):
'''open or jump to next'''
if self.isHidden():
self.disp()
else:
self.findText()
def getText(self):
return self.text().lower()
def keyPressEvent(self, event):
# find the text on enter
if event.key() == Qt.Key_Return or event.key() == Qt.Key_Enter:
self.first = True
self.findText()
return
# clear or hide the searchbar
if event.key() == Qt.Key_Escape:
if self.text() == "":
self.hide()
self.editor.setFocus()
else:
self.clear()
return
# jump to next
if event.key() == Qt.Key_F3:
self.onF3()
self.editor.setFocus()
return
super(SearchBar, self).keyPressEvent(event)
def findText(self):
'''search the prototxt_editor for text'''
text = self.getText()
if text != "":
self.onTextChange()
doc = self.editor.document()
# first search from cursor position
hig = doc.find(text, self.editor.textCursor())
if not hig.isNull():
self.editor.setTextCursor(hig)
return
# if not found: search from start
hig = doc.find(text, QTextCursor(doc))
if not hig.isNull():
self.editor.setTextCursor(hig)
return
# not found, make red
self.setStyleSheet("background:#FFaaaa;")
def updatePosition(self):
'''set position of searchbar'''
rect = self.editor.rect()
dx = 0
if self.editor.verticalScrollBar().isVisible():
dx = self.editor.verticalScrollBar().width()
mx = rect.width() - self.width() - dx - 1
self.move(mx, 2)
def resizeToFont(self):
'''set font of searchbar'''
font = self.font()
fm = QFontMetrics(font)
# adjust height to fontsize
self.setFixedSize(300, fm.height() * 1.2)
def paintEvent(self, event):
self.updatePosition()
super(SearchBar, self).paintEvent(event)
def onTextChange(self):
# reset the style back to default
self.setStyleSheet(self.defaultstyle)
Barista/gui/prototxt_editor/editor_side_bar.py 0000664 ڊ0011117 00000000602 13265130053 022107 0 ustar klemms o0cvpr from PyQt5.QtWidgets import QWidget
from PyQt5.QtCore import QSize
class Sidebar(QWidget):
# constructor class for sidebar
def __init__(self, parent):
QWidget.__init__(self, parent)
self.editor = parent
def sizeHint(self):
return QSize(self.editor.sidebarWidth(), 0)
def paintEvent(self, event):
self.editor.sidebarPaintEvent(event)
Barista/gui/prototxt_editor/editor_slider.py 0000664 ڊ0011117 00000003160 13265130053 021623 0 ustar klemms o0cvpr from PyQt5.QtWidgets import QWidget, QSlider, QHBoxLayout
from PyQt5.QtCore import QMargins
from PyQt5.QtCore import QObject, pyqtSignal
class EditorSlider(QWidget):
# widget containig the slider
# direct subclassing of QSlider leads to errors
valueChanged = pyqtSignal(int)
def __init__(self, vmin, vcur, vmax, parent):
QWidget.__init__(self, parent)
self.editor = parent
self.setFocusPolicy(0)
#layouts
self.layout = QHBoxLayout(self)
self.layout.setContentsMargins(0, 0, 0, 0)
#add the slider
self.slider = QSlider()
self.slider.setRange(vmin, vmax)
self.slider.setValue(vcur)
self.slider.setOrientation(1)
self.slider.setFocusPolicy(0)
self.layout.addWidget(self.slider)
self.slider.valueChanged.connect(self._passThrough)
def updatePosition(self):
'''set the position of the slider'''
rect = self.editor.rect()
dy = 0
dx = 0
#check for scrollbars
if self.editor.verticalScrollBar().isVisible():
dx = self.editor.verticalScrollBar().width()
if self.editor.horizontalScrollBar().isVisible():
dy = self.editor.horizontalScrollBar().height()
mx = rect.width() - self.width() - dx - 5
my = rect.height() - self.height() - dy - 5
self.move(mx, my)
def paintEvent(self, event):
self.updatePosition()
super(EditorSlider, self).paintEvent(event)
def _passThrough(self, value):
'''pass through the valueChanged signal of the QSlider'''
self.valueChanged.emit(value)
Barista/gui/prototxt_editor/editor_syntax_highlight.py 0000664 ڊ0011117 00000011423 13265130053 023717 0 ustar klemms o0cvpr from PyQt5.QtGui import QSyntaxHighlighter, QColor, QTextCharFormat, QFont
from PyQt5.QtCore import QRegExp
import backend.caffe.proto_info as info
class EditorSyntaxHighlighter(QSyntaxHighlighter):
'''
Based on
http://doc.qt.io/qt-5/qtwidgets-richtext-syntaxhighlighter-example.html
https://wiki.python.org/moin/PyQt/Python%20syntax%20highlighting
'''
def __init__(self, document):
QSyntaxHighlighter.__init__(self, document)
self.rules = []
self.enums = []
self.params = []
self.paramgroups = []
self._setupRules()
def _setupRules(self):
# strings
self.tri_single = self._createRule('"', "darkorange", "", 1)
self.tri_double = self._createRule("'", "darkorange", "", 2)
# numbers
self.rules.append(self._createRule('\\b\\d+(\\.\\d+)?(e(\\-|\\+?)\\d+)?\\b', "darkmagenta"))
# get the parameters from caffe
self._extractParams()
# enums
for e in self.enums:
self.rules.append(self._createRule("\\b" + e + "\\b", "cornflowerblue", "bold"))
# parameter
for p in self.params:
self.rules.append(self._createRule("\\b" + p + "\\b", "darkcyan", ""))
# parameter groups
for g in self.paramgroups:
self.rules.append(self._createRule("\\b" + g + "\\b", "indigo", "italic"))
def highlightBlock(self, p_str):
'''apply rules to block'''
# for each rule
for exp, n, form in self.rules:
# find in text
index = exp.indexIn(p_str, 0)
while index >= 0:
index = exp.pos(n)
l = len(exp.cap(n))
# change format till end of string
self.setFormat(index, l, form)
# search for next
index = exp.indexIn(p_str, index + l)
self.setCurrentBlockState(0)
# multiline handling for strings
multiline = self.match_multiline(p_str, *self.tri_single)
if not multiline:
self.match_multiline(p_str, *self.tri_double)
def match_multiline(self, text, exp, n, style):
'''handling of multiline matches'''
# start from beginning
start = 0
add = 0
# if not previously found search anew
if self.previousBlockState() != n:
start = exp.indexIn(text)
add = exp.matchedLength()
while start >= 0:
# find the end
end = exp.indexIn(text, start + add)
if end >= add:
length = end - start + add + exp.matchedLength()
self.setCurrentBlockState(0)
else:
self.setCurrentBlockState(n)
length = len(text) - start + add
# set style
self.setFormat(start, length, style)
# find next
start = exp.indexIn(text, start + length)
if self.currentBlockState() == n:
return True
else:
return False
def _createRule(self, regex, color="black", style="", index=0):
return QRegExp(regex), index, self._format(color, style)
def _format(self, color, style):
# refer to https://www.w3.org/TR/SVG/types.html#ColorKeywords for colors
# create color
_c = QColor()
_c.setNamedColor(color)
# create font
_f = QTextCharFormat()
_f.setForeground(_c)
# set font style
if 'bold' in style:
_f.setFontWeight(QFont.Bold)
if 'italic' in style:
_f.setFontItalic(True)
return _f
def _extractParams(self):
'''open caffe meta information'''
info.resetCaffeProtoModulesvar()
info.CaffeMetaInformation().updateCaffeMetaInformation()
allSolver = info.CaffeMetaInformation().availableSolverTypes()
for solver in allSolver:
self._iterParams(allSolver[solver].parameters())
allLayer = info.CaffeMetaInformation().availableLayerTypes()
for layer in allLayer:
self._iterParams(allLayer[layer].parameters())
def _iterParams(self, dict):
'''recursive param search'''
for param in dict:
obj = dict[param]
# sort
if obj.isParameterGroup():
if param not in self.paramgroups:
self.paramgroups.append(param)
self._iterParams(obj.parameter())
elif obj.isEnum():
if param not in self.params:
self.params.append(param)
values = obj.availableValues()
for v in values:
if v not in self.enums:
self.enums.append(v)
else:
if param not in self.params:
self.params.append(param)
Barista/gui/prototxt_editor/editor_widget.py 0000664 ڊ0011117 00000007447 13265130053 021640 0 ustar klemms o0cvpr from PyQt5.QtCore import pyqtSignal, Qt
from PyQt5.QtWidgets import QDialog, QPushButton, QMessageBox, QAction, QGridLayout
from gui.prototxt_editor import text_edit
class EditorWidget(QDialog):
'''This Editor enables text-based changes of Prototxt files. Connect to the sinal sgSave to get the savedata'''
def __init__(self, syntax = True, parent=None):
# main_window settings
QDialog.__init__(self, parent)
self.setModal(True)
self.resize(600, 600)
self.setMinimumWidth(500)
self.layout = QGridLayout(self)
# buttons
self.pbSave = QPushButton("Save")
self.layout.addWidget(self.pbSave, 0, 0)
self.pbSaveClose = QPushButton("Save && Close")
self.layout.addWidget(self.pbSaveClose, 0, 1)
self.pbClose = QPushButton("Close")
self.layout.addWidget(self.pbClose, 0, 2)
# the prototxt_editor itself
if syntax:
self.textwidget = text_edit.EditorSyntax()
else:
self.textwidget = text_edit.Editor()
self.layout.addWidget(self.textwidget, 1, 0, 1, 3)
self.containsChanges = False
# connections
self.pbSave.clicked.connect(self._onSave)
self.pbSaveClose.clicked.connect(self._onSaveClose)
self.pbClose.clicked.connect(self._onClose)
self.textwidget.textChanged.connect(self._onChange)
# actions
self.saveaction = QAction(self)
self.saveaction.setShortcut('Ctrl+S')
self.addAction(self.saveaction)
self.saveaction.triggered.connect(self._onSave)
self.quitaction = QAction(self)
self.quitaction.setShortcut('Ctrl+Q')
self.addAction(self.quitaction)
self.quitaction.triggered.connect(self._onClose)
sgSave = pyqtSignal(['QString'])
sgClose = pyqtSignal()
def closeEvent(self, event):
'''if closing with changes ask if changes should be discarded'''
if self.containsChanges:
if self._showWarning():
self.sgClose.emit()
event.accept()
else:
event.ignore()
else:
self.sgClose.emit()
event.accept()
def getText(self):
'''convert prototxt_editor to text'''
return self.textwidget.toPlainText()
def setText(self, insertText):
'''replace the current content of the prototxt_editor with the new one'''
self.textwidget.setPlainText(insertText)
self.containsChanges = False
def clear(self):
'''clear the prototxt_editor'''
self.textwidget.clear()
self.textwidget.setFocus()
self.containsChanges = False
def _onSave(self):
'''emit save sinal'''
self.sgSave.emit(self.getText())
self.containsChanges = False
def _onSaveClose(self):
'''save and close'''
self._onSave()
self.close()
def _onClose(self):
self.close()
def _onChange(self):
'''set echange flag'''
self.containsChanges = True
def _showWarning(self):
'''show message box if changes are discarded'''
if QMessageBox.warning(
self, "Warning", "Close without saving?",
QMessageBox.Yes, QMessageBox.No) \
== QMessageBox.Yes:
return True
else:
return False
# workaround
# previously pressing the ESC key would close the main_window and always discard changes
# therefor catch the key and close the main_window manualy
def keyPressEvent(self, event):
if event.key() == Qt.Key_Escape:
self.close()
return
super(EditorWidget, self).keyPressEvent(event)
def disableEditing(self, disable):
''' Set textwidget in read-only mode '''
self.textwidget.setReadOnly(disable)
Barista/gui/prototxt_editor/text_edit.py 0000664 ڊ0011117 00000012167 13265130053 020773 0 ustar klemms o0cvpr from PyQt5.QtCore import QRect, Qt
from PyQt5.QtGui import QFont, QFontMetrics, QColor, QPainter, QFontDatabase
from PyQt5.QtWidgets import QPlainTextEdit
from gui.prototxt_editor import editor_side_bar, editor_search_bar, editor_syntax_highlight
from gui.prototxt_editor import editor_slider
class Editor(QPlainTextEdit):
'''based on http://doc.qt.io/qt-5/qtwidgets-widgets-codeeditor-example.html'''
def __init__(self, parent=None):
QPlainTextEdit.__init__(self, parent)
# default settings
self._fontsize = 12
self._fmin = 8
self._fmax = 30
# setup of prototxt_editor
self.setLineWrapMode(0)
# loading of subwidgets
self.sidebar = editor_side_bar.Sidebar(self) # line numbers
self.searchbar = editor_search_bar.SearchBar(self, self) # search
self.slider = editor_slider.EditorSlider(self._fmin, self._fontsize, self._fmax, self) # fontsize slider
# conntions
self.slider.valueChanged.connect(self.setFontsize)
self.blockCountChanged.connect(self._updateSidebarWidth)
self.updateRequest.connect(self._updateSidebar)
# show
self._updateFont()
self._updateSidebarWidth()
self.sidebar.show()
def setFontsize(self, value):
'''set the fontsize'''
if self._fmin <= value <= self._fmax:
self._fontsize = value
self._updateFont()
def sidebarWidth(self):
'''calculate the width of the sidebar'''
# get the number of digits displayed
digits = 1
maxd = max(1, self.blockCount())
while maxd >= 10:
maxd /= 10
digits += 1
# calc the width
space = 3 + QFontMetrics(self._font).width(*'9') * digits
return space
def _updateSidebarWidth(self, arg=0):
'''refresh the width of the sidebar'''
self.setViewportMargins(self.sidebarWidth(), 0, 0, 0)
def _updateSidebar(self, rect, dy):
'''realign the sidebar'''
if dy is not 0:
self.sidebar.scroll(0, dy)
else:
self.sidebar.update(0, rect.y(), self.sidebarWidth(), rect.height())
if rect.contains(self.viewport().rect()):
self._updateSidebarWidth()
def sidebarPaintEvent(self, event):
'''draw the sidebar'''
# define the colors
_c1 = QColor()
_c1.setNamedColor("lightgrey")
_c2 = QColor()
_c2.setNamedColor("darkgrey")
_c3 = QColor()
_c3.setNamedColor("black")
# setup the painter
painter = QPainter(self.sidebar)
painter.fillRect(event.rect(), _c1)
painter.setFont(self._font)
# calculate the lines
block = self.firstVisibleBlock()
blocknumber = block.blockNumber()
btop = self.blockBoundingGeometry(block).translated(self.contentOffset()).top()
bbottom = btop + self.blockBoundingRect(block).height()
# write the line numbers
while block.isValid() and btop <= event.rect().bottom():
if block.isVisible() and bbottom >= event.rect().top():
number = str(blocknumber + 1)
# change color for "current" line
if self.textCursor().block().blockNumber() != blocknumber:
painter.setPen(_c2)
else:
painter.setPen(_c3)
painter.drawText(0, btop, self.sidebar.width(), QFontMetrics(self._font).height(), 2, number)
block = block.next()
btop = bbottom
bbottom = btop + self.blockBoundingRect(block).height()
blocknumber += 1
def resizeEvent(self, event):
'''update all subwidgets on resize of the main_window'''
super(Editor, self).resizeEvent(event)
self.slider.updatePosition()
self.searchbar.updatePosition()
c = self.contentsRect()
self.sidebar.setGeometry(QRect(c.left(), c.top(), self.sidebarWidth(), c.height()))
def _updateFont(self):
'''set the font and fontsize across all widgets'''
self._font = QFontDatabase.systemFont(QFontDatabase.FixedFont) # QFont("DejaVu Sans Mono", self._fontsize)
self._font.setPointSize(self._fontsize) # Change Fontsize if using QFontDatabase
self.setFont(self._font)
self.searchbar.setFont(self._font)
self.searchbar.resizeToFont()
self.update()
def keyPressEvent(self, event):
# search
if event.key() == Qt.Key_F:
if event.modifiers() & Qt.ControlModifier:
self.searchbar.disp()
return
if event.key() == Qt.Key_F3:
self.searchbar.onF3()
return
# close
if event.key() == Qt.Key_Escape:
if not self.searchbar.isHidden():
self.searchbar.hide()
return
super(Editor, self).keyPressEvent(event)
class EditorSyntax(Editor):
def __init__(self, parent = None):
Editor.__init__(self, parent)
self.highlight = editor_syntax_highlight.EditorSyntaxHighlighter(self.document()) # syntax highlighting
self.highlight._extractParams()
Barista/apps/interactive_digit_classification.py 0000664 ڊ0011117 00000012035 13265130053 022452 0 ustar klemms o0cvpr #! /usr/bin/env python
"""
This is a small self-contained application that demonstrates usage of networks deployed from Barista
in other applications. If net definition and weight's files obtained by training the caffe mnist example
are supplied via command line parameters, the user can draw digits [0-9] in the window, use the net
to classify the result and print the result to stdout.
- Drawing works by pressing the left mouse button and moving the mouse
- The "Return"-Button grabs the image and starts the classification
- The "Backspace"-Button clears the image.
"""
#
#
# Python package dependencies:
# pygame
# caffe
# numpy
import pygame
import caffe
import numpy as np
import argparse
# Argument parsing
parser = argparse.ArgumentParser(description='Interactively classify handwritten digits using neural nets.', epilog=__doc__, formatter_class=argparse.RawTextHelpFormatter)
parser.add_argument('-m', '--model', help='Model (.prototxt) file', type=str, required=True)
parser.add_argument('-w', '--weights', help='Weights (.caffemodel) file', type=str, required=True)
args = parser.parse_args()
#######################################################
# Setup caffe for classification
#######################################################
#load the model
net = caffe.Net(args.model, args.weights, caffe.TEST)
# load input and configure preprocessing
transformer = caffe.io.Transformer({'data': net.blobs['data'].data.shape})
transformer.set_transpose('data', (2,0,1))
transformer.set_raw_scale('data', 1.0)
# Change the batch size to a single image
net.blobs['data'].reshape(1,1,28,28)
#######################################################
# Proprecessing helper functions
#######################################################
# Find the required offset (i.e. difference between center of weight and center of the image)
def find_offset(grid):
x_acc = 0;
y_acc = 0;
h = grid.shape[0]
w = grid.shape[1]
num_points = 0;
for y in np.arange(h):
for x in np.arange(w):
val = (grid[y,x] > 0)
x_acc += (x-w/2.0) * val
y_acc += (y-h/2.0) * val
if val:
num_points += 1
if num_points == 0:
return (0,0)
x_acc /= num_points
y_acc /= num_points
return (y_acc, x_acc)
# Shift and resample values in grid and thus centering the center of weight in the center of the image
def shift(grid):
offset = find_offset(grid)
h = grid.shape[0]
w = grid.shape[1]
image = np.zeros((h, w, 1))
for y in np.arange(h):
for x in np.arange(w):
x_n = int(np.round(x+offset[1]))
y_n = int(np.round(y+offset[0]))
if x_n < 0 or x_n >= w:
val = 0
elif y_n < 0 or y_n >= h:
val = 0
else:
val = grid[y_n,x_n]
image[y,x] = val
return image
# Classify a given image and output the index of the class with the highest probability according to the net and caffe
def classify(pixels):
image = np.zeros((pixels.shape[0], pixels.shape[1], 1))
image[:,:,0] = pixels[:,:]
image = np.transpose(image, (1,0,2))
image = shift(image)
data = np.asarray([transformer.preprocess('data', image)])
out = net.forward_all(data = data)
prob = out['probabilities']
cls = prob.argmax()
return cls
#######################################################
# Pygame application stuff
#######################################################
# Create screen of specified size
screen_size = (112,112)
screen = pygame.display.set_mode(screen_size)
# Global variables/constants used for drawing
currently_drawing = False
last_pos = (0, 0)
draw_color = (255, 255, 255)
clear_color = (0, 0, 0)
brush_radius = 3
# draw a line of circles from start to finish
def roundline(srf, color, start, end, radius):
dx = end[0]-start[0]
dy = end[1]-start[1]
distance = max(abs(dx), abs(dy))
for i in range(distance):
x = int( start[0]+float(i)/distance*dx)
y = int( start[1]+float(i)/distance*dy)
pygame.draw.circle(srf, color, (x, y), radius)
try:
while True:
e = pygame.event.wait()
if e.type == pygame.QUIT:
raise StopIteration
if e.type == pygame.MOUSEBUTTONDOWN:
pygame.draw.circle(screen, draw_color, e.pos, brush_radius)
currently_drawing = True
if e.type == pygame.MOUSEBUTTONUP:
currently_drawing = False
if e.type == pygame.MOUSEMOTION:
if currently_drawing:
pygame.draw.circle(screen, draw_color, e.pos, brush_radius)
roundline(screen, draw_color, e.pos, last_pos, brush_radius)
last_pos = e.pos
if e.type == pygame.KEYDOWN:
if e.key == pygame.K_RETURN:
array = pygame.PixelArray(screen)
cls = classify(array)
print("The net says says: {}".format(cls))
if e.key == pygame.K_BACKSPACE:
screen.fill(clear_color)
pygame.display.flip()
except StopIteration:
pass
pygame.quit()
Barista/backend/__init__.py 0000664 ڊ0011117 00000000000 13265130053 016052 0 ustar klemms o0cvpr Barista/gui/__init__.py 0000664 ڊ0011117 00000000000 13265130053 015247 0 ustar klemms o0cvpr Barista/gui/caffepath_dialog.py 0000664 ڊ0011117 00000027515 13265130053 016774 0 ustar klemms o0cvpr # -*- coding: utf-8 -*-
from PyQt5 import QtWidgets, QtCore
from PyQt5.QtGui import QIcon
from backend.barista import caffe_versions
from gui.host_manager.remote_file_dialog import RemoteFileDialog
from backend.networking.net_util import sendMsgToHost
from backend.networking.protocol import Protocol
from enum import Enum
import sys
import signal
import os.path
class Platform(Enum):
LINUX = 1
WINDOWS = 2
CYGWIN = 3
MACOSX = 4
OS2 = 5
OS2EMX = 6
RISCOS = 7
ATHEOS = 8
class CaffepathDialog(QtWidgets.QDialog):
def __init__(self, reason, buttontext, quitonclose = False, parent = None, remote = False, host = None):
super(CaffepathDialog, self).__init__(parent)
self.setWindowFlags(self.windowFlags() & ~QtCore.Qt.WindowContextHelpButtonHint)
self.setWindowTitle("Caffepath")
self.changes = False
self.quitonclose = quitonclose
self.startPath = "/home"
self.remote = remote
self.host = host
caffepathttext = reason
if remote:
msg = {"key": Protocol.GETPLATFORM}
reply = sendMsgToHost(self.host.host, self.host.port, msg)
platform = reply["platform"]
else:
platform = sys.platform
if platform == "win32": self.platform = Platform.WINDOWS
elif platform == "cygwin": self.platform = Platform.CYGWIN
elif platform == "darwin": self.platform = Platform.MACOSX
elif platform == "os2": self.platform = Platform.OS2
elif platform == "os2emx": self.platform = Platform.OS2EMX
elif platform == "riscos": self.platform = Platform.RISCOS
elif platform == "atheos": self.platform = Platform.ATHEOS
else: self.platform = Platform.LINUX
if remote==True and self.host==None:
print("Error: CaffepathDialog needs a host if remote is True")
exit(-1)
lay = QtWidgets.QGridLayout(self)
lay.addWidget(QtWidgets.QLabel(caffepathttext, self),0,0,1,1)
self.pbSaveStart = QtWidgets.QPushButton(buttontext)
lay.addWidget(self.pbSaveStart, 1, 0, 1, 2)
self.labelName = QtWidgets.QLabel("Unique name:")
lay.addWidget(self.labelName, 3, 0, 1, 4)
self.LineEditName = QtWidgets.QLineEdit()
lay.addWidget(self.LineEditName, 4, 0, 1, 2)
self.labelPathtoCaffe = QtWidgets.QLabel("Path to caffe root:")
lay.addWidget(self.labelPathtoCaffe, 5, 0, 1, 4)
self.LineEditPathtoCaffe = QtWidgets.QLineEdit()
lay.addWidget(self.LineEditPathtoCaffe, 6, 0, 1, 2)
self.labelPathtoPython = QtWidgets.QLabel("Path to python packages of caffe:")
lay.addWidget(self.labelPathtoPython, 7, 0, 1, 4)
self.LineEditPathtoPython = QtWidgets.QLineEdit()
lay.addWidget(self.LineEditPathtoPython, 8, 0, 1, 2)
if self.platform == Platform.WINDOWS:
self.labelPathtoBinaries = QtWidgets.QLabel("Caffe executable:")
else:
self.labelPathtoBinaries = QtWidgets.QLabel("Caffe binary file:")
lay.addWidget(self.labelPathtoBinaries, 9, 0, 1, 4)
self.LineEditPathtoBinaries = QtWidgets.QLineEdit()
lay.addWidget(self.LineEditPathtoBinaries, 10, 0, 1, 2)
self.labelPathtoProto = QtWidgets.QLabel("Caffe proto file:")
lay.addWidget(self.labelPathtoProto, 11, 0, 1, 4)
self.LineEditPathtoProto = QtWidgets.QLineEdit()
lay.addWidget(self.LineEditPathtoProto, 12, 0, 1, 2)
pix = QtWidgets.qApp.style().standardPixmap(QtWidgets.QStyle.SP_DialogOpenButton)
ico = QIcon(pix)
self.pbCaffePath = QtWidgets.QPushButton(ico, "")
lay.addWidget(self.pbCaffePath, 6, 3, 1, 1)
pix = QtWidgets.qApp.style().standardPixmap(QtWidgets.QStyle.SP_DialogOpenButton)
ico = QIcon(pix)
self.pbPythonFile = QtWidgets.QPushButton(ico, "")
lay.addWidget(self.pbPythonFile, 8, 3, 1, 1)
pix = QtWidgets.qApp.style().standardPixmap(QtWidgets.QStyle.SP_DialogOpenButton)
ico = QIcon(pix)
self.pbBinaryFiles = QtWidgets.QPushButton(ico, "")
lay.addWidget(self.pbBinaryFiles, 10, 3, 1, 1)
pix = QtWidgets.qApp.style().standardPixmap(QtWidgets.QStyle.SP_DialogOpenButton)
ico = QIcon(pix)
self.pbProtoFile = QtWidgets.QPushButton(ico, "")
lay.addWidget(self.pbProtoFile, 12, 3, 1, 1)
self.pbCaffePath.clicked.connect(self._onCaffePath)
self.pbPythonFile.clicked.connect(self._onPythonFile)
self.pbBinaryFiles.clicked.connect(self._onBinaryFile)
self.pbProtoFile.clicked.connect(self._onProtoFile)
self.pbSaveStart.clicked.connect(self._onSave)
def _onCaffePath(self):
"""if default path should be changed, show file dialog to chose a path """
ret = ""
if self.remote:
rfd = RemoteFileDialog(self.host.host, self.host.port, "Select rootpath of caffe", dirSelect=True)
rfd.exec_()
ret = rfd.returnvalue
else:
ret = QtWidgets.QFileDialog.getExistingDirectory(self, "Select rootpath of caffe",
self.startPath,
QtWidgets.QFileDialog.ShowDirsOnly)
if len(ret) != 0:
self.LineEditPathtoCaffe.setText(ret)
self.startPath = ret
if len(str(self.LineEditPathtoPython.text())) == 0:
pythonpath = os.path.join(ret, "python")
if os.path.exists(pythonpath):
self.LineEditPathtoPython.setText(pythonpath)
if len(str(self.LineEditPathtoBinaries.text())) == 0:
# There are differnt variations on the name of the caffe binary, so we look for and accept any one of them
for binaryName in ["caffe.exe", "caffe", "caffe.bin"]:
binarypath = os.path.join(ret, "build", "tools", binaryName)
if os.path.isfile(binarypath):
self.LineEditPathtoBinaries.setText(binarypath)
break
if len(str(self.LineEditPathtoProto.text())) == 0:
protopath = os.path.join(ret, "src", "caffe", "proto", "caffe.proto")
if os.path.exists(protopath):
self.LineEditPathtoProto.setText(protopath)
def _onPythonFile(self):
"""if default path should be changed, show file dialog to chose a path """
ret = ""
if self.remote:
rfd = RemoteFileDialog(self.host.host, self.host.port, "Select path to the python packages of caffe", dirSelect=True)
rfd.exec_()
ret = rfd.returnvalue
else:
ret = QtWidgets.QFileDialog.getExistingDirectory(self, "Select path to the python packages of caffe",
self.startPath,
QtWidgets.QFileDialog.ShowDirsOnly)
if len(ret) != 0:
self.LineEditPathtoPython.setText(ret)
def _onBinaryFile(self):
"""if default path should be changed, show file dialog to chose a path """
ret = ""
if self.remote:
if self.platform == Platform.WINDOWS:
rfd = RemoteFileDialog(self.host.host, self.host.port, "Select caffe executable", "ALL (*.exe)", dirSelect=False)
else:
rfd = RemoteFileDialog(self.host.host, self.host.port, "Select caffe binary file", "ALL (*.bin)", dirSelect=False)
rfd.exec_()
ret = rfd.returnvalue
else:
if self.platform == Platform.WINDOWS:
ret = QtWidgets.QFileDialog.getOpenFileName(self, "Select caffe executable", self.startPath, "Executables (*.exe)")
else:
ret = QtWidgets.QFileDialog.getOpenFileName(self, "Select caffe binary file", self.startPath, "Binary Files (*.bin)")
if len(ret) != 0:
self.LineEditPathtoBinaries.setText(ret[0])
def _onProtoFile(self):
"""if default path should be changed, show file dialog to chose a path """
ret = ""
if self.remote:
rfd = RemoteFileDialog(self.host.host, self.host.port, "Select caffe proto file", "ALL (*.proto)", dirSelect=False)
rfd.exec_()
ret = rfd.returnvalue
else:
ret = QtWidgets.QFileDialog.getOpenFileName(self, "Select caffe proto file", self.startPath, "Proto Files (*.proto)");
if len(ret) != 0:
self.LineEditPathtoProto.setText(ret[0])
def _onSave(self):
"""if path in LineEditPathtoPython is correct caffepath, change caffepath and close window,
otherwise show warning """
caffename = str(self.LineEditName.text())
caffepath = str(self.LineEditPathtoCaffe.text())
pythonpath = str(self.LineEditPathtoPython.text())
binarypath = str(self.LineEditPathtoBinaries.text())
protopath = str(self.LineEditPathtoProto.text())
versions = []
if self.remote:
msg = {"key": Protocol.GETCAFFEVERSIONS}
reply = sendMsgToHost(self.host.host, self.host.port, msg)
if reply:
remoteVersions = reply["versions"]
for version in remoteVersions:
versions.append(caffe_versions.caffeVersion(version, remoteVersions[version]["root"], remoteVersions[version]["binary"], remoteVersions[version]["python"], remoteVersions[version]["proto"]))
else:
versions = caffe_versions.getAvailableVersions()
existing = False
for version in versions:
if version.getName() == caffename:
existing = True
if existing:
QtWidgets.QMessageBox.warning(self, "Error", "An version with the given name already exists. Please choose a unique name.")
else:
try:
if len(caffepath) == 0: raise Exception("Please select the caffe root path")
try:
import importlib
import sys
sys.path.insert(0, pythonpath)
caffe = importlib.import_module("caffe")
sys.path.pop(0)
except ImportError as e:
raise Exception("Please select the path to the python package of caffe")
if len(binarypath) == 0:
if self.platform == Platform.WINDOWS:
raise Exception("Plese select the caffe executable")
else:
raise Exception("Please select the caffe binary file")
if len(protopath) == 0: raise Exception("Please select the proto file")
#if os.path.isfile(binarypath)
version = caffe_versions.caffeVersion(caffename, caffepath, binarypath, pythonpath, protopath)
if self.remote:
msg = {"key": Protocol.ADDCAFFEVERSION, "version": {"name": version.getName(),
"root": version.getRootpath(),
"binary": version.getBinarypath(),
"python": version.getPythonpath(),
"proto": version.getProtopath()}}
sendMsgToHost(self.host.host, self.host.port, msg)
else:
caffe_versions.addVersion(version)
self.changes = True
self.close()
except Exception as e:
QtWidgets.QMessageBox.warning(self, "Error", self.tr(repr(e)))
def closeEvent(self, event):
if(self.quitonclose):
#if nothing has changed(no new Caffepath) just exit complete programm
if not self.changes:
exit(1)
Barista/gui/change_caffe_version_dialog.py 0000664 ڊ0011117 00000007704 13265130053 021167 0 ustar klemms o0cvpr from PyQt5.QtWidgets import (
QDialog,
QLabel,
QPushButton,
QListWidget,
QMessageBox,
QListWidgetItem,
QGridLayout,
QStyle,
qApp,
QAbstractItemView
)
from gui.caffepath_dialog import CaffepathDialog
import backend.barista.caffe_versions as caffeVersions
import os
import json
class CaffeVersionDialog(QDialog):
def __init__(self, exception, dir, default_actions, parent = None):
super(CaffeVersionDialog, self).__init__(parent)
self.exception = exception
self.parent = parent
self.dir = dir
self.actions = default_actions
self.projectVersion = self.getProjectVersion()
self.setWindowTitle("Warning!")
self.layout = QGridLayout(self)
self.lblWarning = QLabel("Failed to load Project with caffe-version: '" + self.projectVersion + "'\n{} \nPlease select another caffe-version for the project".format(str(self.exception)))
self.layout.addWidget(self.lblWarning, 0, 0, 1, 1)
self.pbAddCaffeVersion = QPushButton("Add Version")
self.layout.addWidget(self.pbAddCaffeVersion, 1, 0, 1, 1)
self.lstVersions = QListWidget(self)
self.lstVersions.setSelectionMode(QAbstractItemView.SingleSelection)
self.layout.addWidget(self.lstVersions, 2, 0, 1, 1)
self.pbChangeCaffeVersion = QPushButton("Select")
self.layout.addWidget(self.pbChangeCaffeVersion, 3, 0, 1, 1)
self.pbAddCaffeVersion.clicked.connect(self._onAddVersion)
self.pbChangeCaffeVersion.clicked.connect(self._onSetProjectDefault)
self.lstVersions.itemSelectionChanged.connect(self._onSelectionChanged)
self.updateList()
def _onAddVersion(self):
"""Opens the dialog to add new caffe versions"""
caffedlg = CaffepathDialog("Add a new caffe version to Barista", "Add version")
caffedlg.exec_()
self.updateList()
def updateList(self):
"""updates the list containing all available caffe versions"""
self.lstVersions.clear()
versions = caffeVersions.getAvailableVersions()
for version in versions:
name = version.getName()
lblVersion = QLabel(name)
item = QListWidgetItem()
self.lstVersions.addItem(item)
self.lstVersions.setItemWidget(item, lblVersion)
def _onSelectionChanged(self):
if len(self.lstVersions.selectedItems()) < 1:
self.pbChangeCaffeVersion.setEnabled(False)
else:
self.pbChangeCaffeVersion.setEnabled(True)
def getProjectVersion(self):
configfile = os.path.join(self.dir, "barista_project.json")
with open(configfile, "r") as file:
res = json.load(file)
file.close()
if "caffeVersion" in res:
return res["caffeVersion"]
else:
return None
def _onSetProjectDefault(self, name):
"""Sets the default caffe version of the current project"""
caffeVersions.loadVersions()
configfile = os.path.join(self.dir, "barista_project.json")
with open(configfile, "r") as file:
res = json.load(file)
file.close()
if "caffeVersion" in res:
caffeVersion = self.lstVersions.itemWidget(self.lstVersions.currentItem()).text()
res["caffeVersion"] = caffeVersion
with open(configfile, "w") as file:
json.dump(res, file, sort_keys=True, indent=4)
file.close()
msgBox = QMessageBox(QMessageBox.Warning, "Warning", "Please restart Barista client for changes to apply, otherwise Barista may be unstable!")
msgBox.addButton("Ok", QMessageBox.NoRole)
msgBox.addButton("Restart now", QMessageBox.YesRole)
ret = msgBox.exec_()
if ret == 1:
self.actions.restart(self.dir)
self.close()
def closeEvent(self, event):
if not self.parent.isVisible():
exit()
Barista/gui/deployment_dialog.py 0000664 ڊ0011117 00000030321 13265130053 017220 0 ustar klemms o0cvpr # -*- coding: utf-8 -*-
import os
import ntpath
from PyQt5 import QtWidgets
from backend.barista.utils.logger import Log
from backend.barista.deployed_net import DeployedNet
from backend.barista.project import Project
from backend.barista.session.session_utils import Paths
class DeploymentDialog(QtWidgets.QDialog):
COMBO_INDEX_INLINE = 0
COMBO_INDEX_SEPARATE_FILE = 1
def __init__(self, sessions, parent = None):
super(DeploymentDialog, self).__init__(parent)
# Save given params.
self._sessions = sessions
self._setupUi()
self.caller_id = None
def _setupUi(self):
self.setWindowTitle(self.tr("Deployment - Options"))
mainLayout = QtWidgets.QVBoxLayout(self)
# browse destination folder
mainLayout.addWidget(QtWidgets.QLabel(self.tr("Export deployment files to"), self))
self._textDestination = QtWidgets.QLineEdit()
btnBrowseDestination = QtWidgets.QPushButton(self.tr("Browse"))
btnBrowseDestination.clicked.connect(lambda: self.selectFolder())
layout = QtWidgets.QHBoxLayout()
layout.addWidget(self._textDestination)
layout.addWidget(btnBrowseDestination)
mainLayout.addLayout(layout)
# snapshot selection
self._snapshotInfoByComboIndex = []
mainLayout.addWidget(QtWidgets.QLabel(self.tr("Snapshot (determines caffemodel file and "
" network version)."), self))
self._comboNet = QtWidgets.QComboBox()
for sessionKey in sorted(self._sessions, reverse=True):
session = self._sessions[sessionKey]
snapshots = session.getSnapshots()
for snapshotKey in sorted(snapshots, reverse=True):
snapshot = snapshots[snapshotKey]
self._comboNet.addItem(self.tr("Session " + str(session.getSessionId()) + " - " +
os.path.basename(snapshot)))
self._snapshotInfoByComboIndex.append({"session": session,
"snapshot": snapshot})
mainLayout.addWidget(self._comboNet)
# bottom button bar
layout = QtWidgets.QHBoxLayout()
btnDeployAndExport = QtWidgets.QPushButton(self.tr("Deploy and Export"))
btnDeployAndExport.clicked.connect(lambda: self.deployAndExport())
btnCancel = QtWidgets.QPushButton(self.tr("Cancel"))
btnCancel.clicked.connect(lambda: self.cancel())
layout.addWidget(btnCancel)
layout.addWidget(btnDeployAndExport)
mainLayout.addLayout(layout)
# Fix width and height.
self.layout().setSizeConstraint(QtWidgets.QLayout.SetFixedSize)
def getCallerId(self):
""" Return the unique caller id for this session
"""
if self.caller_id is None:
self.caller_id = Log.getCallerId('Deployment')
return self.caller_id
def selectFolder(self):
"""
Opens a directory selection dialog and sets the line edit's value to the
selected path. This will be triggered when the user clicks the browse button.
"""
dir_path = QtWidgets.QFileDialog.getExistingDirectory(
parent=self
)
self._textDestination.setText(dir_path)
def cancel(self):
"""
Closes the dialog. This will be triggered when the user clicks the
cancel button.
"""
self.close()
def _replaceLast(self, source_string, replace_what, replace_with):
"""
Replace the very last occurrence of replace_what with replace_with in source_string.
Code taken from: http://stackoverflow.com/a/3675423
"""
head, sep, tail = source_string.rpartition(replace_what)
return head + replace_with + tail
def deployAndExport(self):
"""
Wrapper around _deployAndExportUnsafe that catches exceptions and shows
them to the user in an error dialog.
"""
try:
self._deployAndExportUnsafe()
except Exception as e:
QtWidgets.QMessageBox.critical(self,
self.tr("Can't use specified destination"),
self.tr("Error: " + str(e)))
self.close()
def _deployAndExportUnsafe(self):
"""
This will be triggered when the user clicks the deploy button. It validates
the user input and displays messages and errors if additional input is
required. If everything is validated successfully, the session is exported
to the destination directory.
"""
# If no snapshot exists, we show an error message and close the dialog.
if not self._hasSnapshotsOrDisplayError():
self.close()
return
# Get the destination folder from the current user input. If the input
# is empty, we show an error message and cancel.
destinationFolder = self._getDestinationOrDisplayError()
if not destinationFolder:
return
# Check if the path already exsists. If it doesn't exist yet, let the user
# decide whether to create all missing folders and abort otherwise.
folderExists = os.path.isdir(destinationFolder)
if not folderExists and not self._askDirectoryCreatePermission():
return
# Ensure that the full path points to a folder and not a file.
if not self._ensurePathIsFolderOrDisplayError(destinationFolder):
return
# Determine the destination file paths for
destinationPrototxtFile, caffemodelDestination = self._getDestinationFilePaths(destinationFolder)
# Check if any of the destination files already exsist and ask the user
# if they should be replaced. Abort if the user decides not to replace one
# of the files.
if not self._checkFilesDontExistOrAskReplacePermission([destinationPrototxtFile, caffemodelDestination]):
return
# Export files.
session = self._selectedSession()
snapshot = self._selectedSnapshot()
caffemodelContents = session.readCaffemodelFile(self._replaceLast(snapshot,
'solverstate',
'caffemodel'))
# Start deployment.
deployedNet = session.readDeployedNetAsString()
# Write prototxt file.
with open(destinationPrototxtFile, 'w') as file:
file.write(deployedNet)
# Write caffemodel file.
with open(caffemodelDestination, 'w') as caffemodelFile:
caffemodelFile.write(caffemodelContents)
Log.log("Deployment files have been saved successfully to {}.".format(destinationPrototxtFile), self.getCallerId())
# Close the current dialog.
self.close()
def _selectedSession(self):
""" Returns the session that belongs to the currently selected snapshot. """
return self._snapshotInfoByComboIndex[self._comboNet.currentIndex()]["session"]
def _selectedSnapshot(self):
""" Returns the currently selected snapshot name. """
return self._snapshotInfoByComboIndex[self._comboNet.currentIndex()]["snapshot"]
def _hasSnapshotsOrDisplayError(self):
"""
Displays an error QMessageBox and returns False if no snapshots exist.
Returns True otherwise.
"""
if len(self._snapshotInfoByComboIndex) <= 0:
QtWidgets.QMessageBox.critical(self,
self.tr("Deployment not available"),
self.tr("There needs to be at least one saved snapshot to allow deployment."))
return False
return True
def _getDestinationOrDisplayError(self):
"""
If the user has not input a destination directory, this shows an error
QMessageBox and returns False. Returns the destination path otherwise.
"""
destinationFolder = self._textDestination.text()
if len(destinationFolder) <= 0:
QtWidgets.QMessageBox.critical(self,
self.tr("Can't use specified destination"),
self.tr("Please select a valid destination to export deployment files."))
return False
return destinationFolder
def _ensurePathIsFolderOrDisplayError(self, path):
"""
Checks if the given path is a folder and not a file. Displays an error
QMessageBox if it is not a folder and returns False. Returns True otherwise.
"""
# Ensure that the full path points to a folder and not a file.
destinationIsFolder = not os.path.exists(path) or os.path.isdir(path)
# If it does point to a file, we show an error and return False.
if not destinationIsFolder:
QtWidgets.QMessageBox.critical(self, self.tr("Can't use specified destination"),
self.tr("The given path points to an existing file instead of "
"a folder."))
return False
return True
def _checkFilesDontExistOrAskReplacePermission(self, files):
"""
Takes a list of filepaths as an input and for each of those paths checks
if the file already exists. If it does, it asks the user if it should be
replaced. If the user clicks 'No' on any of the dialogs it returns False.
Returns True otherwise.
"""
for f in files:
fileAlreadyExists = os.path.exists(f)
basename = ntpath.basename(f)
if fileAlreadyExists and not self._askFileReplacePermission(basename):
return False
return True
def _askFileReplacePermission(self, filename):
""",
Shows a QMessageBox that asks the user if the file with name filename should
be replaced. Returns True if the user clicks 'Yes' and False otherwise.
"""
reply = QtWidgets.QMessageBox.question(self,
self.tr("File does already exist."),
self.tr(
"Do you want to replace the existing "
"%s file?" % (filename)),
QtWidgets.QMessageBox.Yes,
QtWidgets.QMessageBox.No)
return reply == QtWidgets.QMessageBox.Yes
def _askDirectoryCreatePermission(self):
"""
Shows a QMessageBox that asks the user if the destination directories should
be created. If the users clicks 'Yes', the directories will be created.
Returns True if the user clicked 'Yes', False otherwise.
"""
reply = QtWidgets.QMessageBox.question(self,
self.tr("Destination does not exist."),
self.tr("Do you want to create all non-existing folders in the given path?"),
QtWidgets.QMessageBox.Yes,
QtWidgets.QMessageBox.No)
# If the user agrees, we create the directory.
if reply == QtWidgets.QMessageBox.Yes:
os.makedirs(destinationFolder)
return reply == QtWidgets.QMessageBox.Yes
def _getDestinationFilePaths(self, directory):
"""
Returns a tuple where the first element is the filepath for the destination
prototxt file and the second element is the filepath for the destination
caffemodel file.
"""
# Determine the destination file paths for
destinationPrototxtFile = os.path.join(directory, "deployed_net.prototxt")
caffemodelFilename = self._replaceLast(self._selectedSnapshot(),
"solverstate", "caffemodel")
caffemodelDestination = os.path.join(os.path.dirname(destinationPrototxtFile),
caffemodelFilename)
return destinationPrototxtFile, caffemodelDestination
Barista/gui/export_solver_dialog.py 0000664 ڊ0011117 00000022256 13265130053 017763 0 ustar klemms o0cvpr # -*- coding: utf-8 -*-
import copy
import os
from PyQt5 import QtWidgets
from backend.barista.utils.logger import Log
from backend.caffe import saver
class ExportSolverDialog(QtWidgets.QDialog):
COMBO_INDEX_INLINE = 0
COMBO_INDEX_SEPARATE_FILE = 1
def __init__(self, solver, network, parent = None, defaultActions=None):
super(ExportSolverDialog, self).__init__(parent)
# create a deep copy of the solver, as we will need to manipulate it a little bit
self._solver = copy.deepcopy(solver.valuedict)
self._network = network
# this will be used to add paths to the recent file list
self._defaultActions = defaultActions
self.PROTOTXT_FILTER = self.tr("Prototxt Files (*.prototxt);;All files (*)")
self.setWindowTitle(self.tr("Export Solver - Options"))
mainLayout = QtWidgets.QVBoxLayout(self)
# browse solver file
mainLayout.addWidget(QtWidgets.QLabel(self.tr("Export solver definition to"), self))
self._textPathSolver = QtWidgets.QLineEdit()
btnBrowseSolver = QtWidgets.QPushButton(self.tr("Browse"))
btnBrowseSolver.clicked.connect(lambda: self.selectFileSolver())
layout = QtWidgets.QHBoxLayout()
layout.addWidget(self._textPathSolver)
layout.addWidget(btnBrowseSolver)
mainLayout.addLayout(layout)
# options to include the net
# pay attention to the order in which the combo items are added.
# they must fit to COMBO_INDEX_SEPARATE_FILE and COMBO_INDEX_INLINE
mainLayout.addWidget(QtWidgets.QLabel(self.tr("Include net definition"), self))
self._comboNet = QtWidgets.QComboBox()
self._comboNet.addItem(self.tr("inline (parameter: net_param)"))
self._comboNet.addItem(self.tr("by pointing to the following file (parameter: net)"))
self._comboNet.currentIndexChanged.connect(self.selectionChange)
mainLayout.addWidget(self._comboNet)
# browse net file
self._textPathNet = QtWidgets.QLineEdit()
self._textPathNet.setDisabled(True)
self._btnBrowseNet = QtWidgets.QPushButton(self.tr("Browse"))
self._btnBrowseNet.clicked.connect(lambda: self.selectFileNet())
self._btnBrowseNet.setDisabled(True)
layout = QtWidgets.QHBoxLayout()
layout.addWidget(self._textPathNet)
layout.addWidget(self._btnBrowseNet)
mainLayout.addLayout(layout)
# bottom button bar
layout = QtWidgets.QHBoxLayout()
btnSave = QtWidgets.QPushButton(self.tr("Save"))
btnSave.clicked.connect(lambda: self.save())
btnCancel = QtWidgets.QPushButton(self.tr("Cancel"))
btnCancel.clicked.connect(lambda: self.cancel())
layout.addWidget(btnCancel)
layout.addWidget(btnSave)
mainLayout.addLayout(layout)
# use fixed size
self.layout().setSizeConstraint(QtWidgets.QLayout.SetFixedSize)
def selectFileNet(self):
filename, _ = QtWidgets.QFileDialog.getOpenFileName(
parent=self,
filter=self.PROTOTXT_FILTER
)
self._textPathNet.setText(filename)
def selectFileSolver(self):
filename, _ = QtWidgets.QFileDialog.getSaveFileName(
parent=self,
filter=self.PROTOTXT_FILTER
)
# if the file does not yet exist and no file extension has been specified,
# set it to the default .prototxt
if len(filename) > 0 and not os.path.exists(filename) and os.path.splitext(filename)[1] == "":
filename += ".prototxt"
self._textPathSolver.setText(filename)
def selectionChange(self, selectedIndex):
disableNetFileSelection = (selectedIndex != self.COMBO_INDEX_SEPARATE_FILE)
self._textPathNet.setDisabled(disableNetFileSelection)
self._btnBrowseNet.setDisabled(disableNetFileSelection)
def cancel(self):
self.close()
def save(self):
try:
# get user input
netFullPath = self._textPathNet.text()
solverFullPath = self._textPathSolver.text()
solverDirPath = os.path.dirname(solverFullPath)
netInSeparateFile = (self._comboNet.currentIndex() == self.COMBO_INDEX_SEPARATE_FILE)
netPathIsValid = len(netFullPath) > 0 and os.path.exists(netFullPath) \
and os.path.isfile(netFullPath)
# ensure that the input isn't empty
if len(solverFullPath) > 0:
# ensure that the path (except the base name) does already exist
folderExists = os.path.isdir(solverDirPath)
# if it doesn't exist yet, let the user decide whether to create all missing folders
if not folderExists:
reply = QtWidgets.QMessageBox.question(self,
self.tr("Destination doesn't exist yet."),
self.tr("Do you want to create all non-existing folders "
"in the given path?"),
QtWidgets.QMessageBox.Yes,
QtWidgets.QMessageBox.No)
if reply == QtWidgets.QMessageBox.Yes:
folderExists = True
os.makedirs(solverDirPath)
if folderExists:
# ensure that the full path does point to a file and not a folder
fileIsNoFolder = not os.path.exists(solverFullPath) or not os.path.isdir(solverFullPath)
# input is valid, go ahead and start the actual export
if fileIsNoFolder:
if netInSeparateFile:
# network path does not need to be valid, as we are not doing anything with the referenced file
# anyway: let the user decide whether an invalid path is used on purpose
if not netPathIsValid:
reply = QtWidgets.QMessageBox.question(self,
self.tr("Network path seems to be invalid."),
"Do you want to continue anyway?",
QtWidgets.QMessageBox.Yes,
QtWidgets.QMessageBox.No)
if reply == QtWidgets.QMessageBox.Yes:
netPathIsValid = True
if netPathIsValid:
# point to selected network file
self._solver["net"] = netFullPath
# remove any other references to a network definition
if "net_param" in self._solver:
del self._solver["net_param"]
else:
# include inline definition of the network
self._solver["net_param"] = self._network
# remove any other references to a network file
if "net" in self._solver:
del self._solver["net"]
# finally, save solver prototxt
if not netInSeparateFile or netPathIsValid:
with open(solverFullPath, 'w') as file:
file.write(saver.saveSolver(self._solver))
callerId = Log.getCallerId('export_solver')
Log.log(
"The solver has been exported successfully to {}.".format(
solverFullPath
), callerId)
# add used file paths to the recent file list
if self._defaultActions is not None:
self._defaultActions.recentSolverData.addRecently(solverFullPath, solverFullPath)
if netInSeparateFile and netPathIsValid:
self._defaultActions.recentNetData.addRecently(netFullPath, netFullPath)
self.close()
else:
QtWidgets.QMessageBox.critical(self, self.tr("Can't save solver"),
self.tr("The given path points to an existing folder instead of "
"a file."))
else:
QtWidgets.QMessageBox.critical(self,
self.tr("Can't save solver"),
self.tr("Please select a valid destination for the solver file."))
except:
QtWidgets.QMessageBox.critical(self,
self.tr("Can't save solver"),
self.tr("Unknown error."))
self.close()
Barista/gui/gui_util.py 0000664 ڊ0011117 00000010005 13265130053 015337 0 ustar klemms o0cvpr from PyQt5 import QtWidgets
from PyQt5.QtCore import Qt, QRegExp
from backend.barista.project import dirIsProject
import os
def askFromList(parent, layerlist, title, text, defaultSelection=True):
'''given a list of layernames, select one'''
# create a QDialog
dialog = QtWidgets.QDialog(parent)
dialog.setWindowTitle(title)
# set some layouts
layout = QtWidgets.QVBoxLayout(dialog)
sublayout = QtWidgets.QHBoxLayout()
# set Widgets
label = QtWidgets.QLabel(text)
layout.addWidget(label)
list = QtWidgets.QListWidget(dialog)
layout.addWidget(list)
# add the listitems
for id, name in layerlist:
item = QtWidgets.QListWidgetItem(list)
item.setData(Qt.UserRole, id)
item.setData(0, name)
list.addItem(item)
# add the buttons
pb_ok = QtWidgets.QPushButton("Ok")
pb_can = QtWidgets.QPushButton("Cancel")
sublayout.addWidget(pb_ok)
sublayout.addWidget(pb_can)
layout.addLayout(sublayout)
# connect the buttons
list.doubleClicked.connect(dialog.accept)
pb_ok.clicked.connect(dialog.accept)
pb_can.clicked.connect(lambda: (list.clearSelection(), dialog.close()))
# select the first item of the list by default
if defaultSelection:
list.setCurrentRow(0)
# exec the dialog and check if return value is 1 or 0
if dialog.exec_() == 0:
return None
# process the selection
selection = list.selectedItems()
if len(selection) == 0:
return None
# return the ID of the selected layer
return selection[0].data(Qt.UserRole)
def getRecentProjectsFolder(defAct):
""" Get the folder containing (!) the most recent project.
defAct: DefaultActions to use
returns: folder path or None
"""
# Find starting directory to start new project dialog
recentProjectTuple = defAct.recentProjectsData.getMostRecent()
if recentProjectTuple is None:
return
else:
[_, mostRecentProjectPath] = recentProjectTuple
return os.path.dirname(mostRecentProjectPath)
def getOpenProjectDir(defAct, parent):
""" Interactively select an existing project directory
defAct: DefaultActions to use
parent: a Qt parent object
returns: folder path (may be empty if selection fails)
"""
directory = QtWidgets.QFileDialog.getExistingDirectory(parent = parent, caption = defAct.tr("Directoy of the project"), directory = getRecentProjectsFolder(defAct))
if len(directory) == 0:
return ""
if not dirIsProject(directory):
answer = QtWidgets.QMessageBox.question(parent, defAct.tr("No Project"), defAct.tr("The selected folder does not seem to be a valid project. Load this dir anyway?"))
if answer == QtWidgets.QMessageBox.No:
return ""
return directory
def getNewProjectDir(defAct, parent):
""" Interactively select a potential new project directory
defAct: DefaultActions to use
parent: a Qt parent object
returns: folder path (may be empty if selection fails)
"""
# Select new project dir path
newProjectDir, _ = QtWidgets.QFileDialog.getSaveFileName(parent = parent, caption = defAct.tr("New Project Directory"), directory = getRecentProjectsFolder(defAct))
if len(newProjectDir) == 0:
return ""
name = os.path.basename(newProjectDir)
# If name contains special characters
regExpName = QRegExp("[A-Za-z0-9_()-][A-Za-z0-9_ #()-]*")
if not regExpName.exactMatch(name):
QtWidgets.QMessageBox.warning(parent, defAct.tr("Invalid character"),
defAct.tr("Project name is invalid. You may only use letters, digits, whitespace and the following special characters: _ # ( ) -"))
return ""
# if the path already exists
if os.path.exists(newProjectDir):
QtWidgets.QMessageBox.warning(parent, defAct.tr("Already exists"),
defAct.tr("Project with this name already exists"))
return ""
# if everything is alright
return newProjectDir
Barista/gui/ipy_widget.py 0000664 ڊ0011117 00000003345 13265130053 015673 0 ustar klemms o0cvpr
# https://github.com/jupyter/qtconsole/blob/master/examples/inprocess_qtconsole.py
# https://stackoverflow.com/questions/26666583/embedding-ipython-qtconsole-and-passing-objects
import sys
qtconsole = True
if qtconsole:
from qtconsole.qt import QtGui
from qtconsole.rich_jupyter_widget import RichJupyterWidget
from qtconsole.inprocess import QtInProcessKernelManager
from ipykernel.inprocess import InProcessKernelManager
def show(parent):
global ipython_widget # Prevent from being garbage collected
# Create an in-process kernel
kernel_manager = QtInProcessKernelManager()
kernel_manager.start_kernel(show_banner=False)
kernel = kernel_manager.kernel
kernel.gui = 'qt'
kernel_client = kernel_manager.client()
kernel_client.start_channels()
kernel_client.namespace = parent
ipython_widget = RichJupyterWidget()
ipython_widget.kernel_manager = kernel_manager
ipython_widget.kernel_client = kernel_client
ipython_widget.show()
kernel.shell.push({"manager": parent})
else:
from IPython.kernel.inprocess import InProcessKernelManager
from IPython.terminal.console.interactiveshell import ZMQTerminalInteractiveShell
#https://github.com/ipython/ipykernel/blob/master/examples/embedding/inprocess_terminal.py
def show(parent):
kernel_manager = InProcessKernelManager()
kernel_manager.start_kernel()
kernel = kernel_manager.kernel
# kernel.gui = 'qt'
kernel.shell.push({"manager": parent})
client = kernel_manager.client()
client.start_channels()
shell = ZMQTerminalInteractiveShell(manager=kernel_manager, client=client)
shell.mainloop()
Barista/gui/manager_dialog.py 0000664 ڊ0011117 00000007516 13265130053 016464 0 ustar klemms o0cvpr from PyQt5 import QtCore, QtWidgets
from PyQt5.QtCore import Qt
from PyQt5.QtWidgets import QWidget, QListWidget
class ManagerDialog(QWidget):
def __init__(self, parent=None):
QWidget.__init__(self, parent)
# remember the parent for future use
self.parent = parent
self.dict = None
# main_window settings
self.setWindowRole("QDialog")
self.setWindowModality(Qt.ApplicationModal)
self.setWindowFlags(self.windowFlags() & QtCore.Qt.CustomizeWindowHint)
self.setWindowFlags(self.windowFlags() | Qt.Window | Qt.WindowMaximizeButtonHint | Qt.WindowCloseButtonHint)
self.setWindowFlags(self.windowFlags() & ~QtCore.Qt.WindowMinimizeButtonHint)
self.resize(800, 600)
# layouts
self._layout = QtWidgets.QVBoxLayout(self)
self._buttonlayout = QtWidgets.QHBoxLayout()
self._layout.addLayout(self._buttonlayout)
# item list widget
self._itemscroll = QListWidget()
def moveUp(self, subDictName, id):
""" move the item one position up """
if id:
i = self.dict[subDictName].index(id)
row = self._getIndexinList(id)
if i > 0 and row > 0:
j = 1
item = self._itemscroll.item(row - j)
# skip hidden items
while i-j > 0 and item.isHidden():
j += 1
item = self._itemscroll.item(row - j)
if not item or item.isHidden():
return
# update dict
tmp = self.dict[subDictName][i]
for k in range(0, j):
self.dict[subDictName][i-k] = self.dict[subDictName][i-k-1]
self.dict[subDictName][i-j] = tmp
widget = self.createWidget(id)
#move item widgets
scroll = self._itemscroll.cursor()
item = self._itemscroll.takeItem(row)
item.setSizeHint(widget.sizeHint())
self._itemscroll.insertItem(row - j, item)
self._itemscroll.setItemWidget(item, widget)
self._itemscroll.setCursor(scroll)
self.updateAfterMovement()
def moveDown(self, subDictName, id):
""" move the item one position down """
if id:
i = self.dict[subDictName].index(id)
row = self._getIndexinList(id)
dictLength = len(self.dict[subDictName])
if i < dictLength - 1 and row < dictLength - 1:
j = 1
item = self._itemscroll.item(row + j)
# skip hidden items
while i+j < dictLength and item.isHidden():
j += 1
item = self._itemscroll.item(row + j)
if not item or item.isHidden():
return
# update dict
tmp = self.dict[subDictName][i]
for k in range(0, j):
self.dict[subDictName][i + k] = self.dict[subDictName][i + k + 1]
self.dict[subDictName][i + j] = tmp
widget = self.createWidget(id)
#move item widgets
scroll = self._itemscroll.cursor()
item = self._itemscroll.takeItem(row)
item.setSizeHint(widget.sizeHint())
self._itemscroll.insertItem(row + j, item)
self._itemscroll.setItemWidget(item, widget)
self._itemscroll.setCursor(scroll)
self.updateAfterMovement()
def createWidget(self, id):
""" abstract method returning a widget for one entry in the list """
raise NotImplementedError("Please Implement this method")
def updateAfterMovement(self):
""" abstract method """
raise NotImplementedError("Please Implement this method")
Barista/gui/start_dialog.py 0000664 ڊ0011117 00000013666 13265130053 016212 0 ustar klemms o0cvpr # -*- coding: utf-8 -*-
from PyQt5 import QtWidgets, QtCore
from PyQt5.QtGui import QIcon
from backend.barista.utils.settings import applicationQSetting
from gui.main_window import main_window
from backend.caffe.proto_info import resetCaffeProtoModulesvar, UnknownLayerTypeException
from gui.main_window.default_actions import DefaultActions, dirIsProject
from backend.barista.project import Project
from backend.barista.session.session import State
from gui_util import getNewProjectDir, getOpenProjectDir
import backend.barista.caffe_versions as caffe_versions
from gui.change_caffe_version_dialog import CaffeVersionDialog
# True => Enable Development Feature like skip project loading
DEVELOP_MODE = False
class StartDialog(QtWidgets.QDialog):
def __init__(self, parent = None):
super(StartDialog, self).__init__(parent)
#reload the caffe und proto Modules from the program settings to get a working version
self.setWindowFlags(self.windowFlags() & ~QtCore.Qt.WindowContextHelpButtonHint)
self.setWindowTitle("Barista")
self.setWindowIcon(QIcon('./resources/icon.png'))
self._defAct = DefaultActions(self)
# self._win = main_window.UiMainWindow()
# self._defAct.projectChanged.connect(self.__switchMainWindow)
lay = QtWidgets.QVBoxLayout(self)
welcomeText="Barista is a graphical user interface for caffe, a deep neural network framework.