Logging improvements with saved files

akvo/gui/akvoGUI.py

@@ -53,27 +53,37 @@ class VectorXr(yaml.YAMLObject):
         return "np.array(%r)" % (self.data)
 
 from collections import OrderedDict
-def represent_ordereddict(dumper, data):
-    value = []
-    for item_key, item_value in data.items():
-        node_key = dumper.represent_data(item_key)
-        node_value = dumper.represent_data(item_value)
-        value.append((node_key, node_value))
-    return yaml.nodes.MappingNode(u'tag:yaml.org,2002:map', value)
-yaml.add_representer(OrderedDict, represent_ordereddict)
+#def represent_ordereddict(dumper, data):
+#    print("representing IN DA HOUSE!!!!!!!!!!!!!!!!!!!!!")
+#    value = []
+#    for item_key, item_value in data.items():
+#        node_key = dumper.represent_data(item_key)
+#        node_value = dumper.represent_data(item_value)
+#        value.append((node_key, node_value))
+#    return yaml.nodes.MappingNode(u'tag:yaml.org,2002:map', value)
+#yaml.add_representer(OrderedDict, represent_ordereddict)
+
+def setup_yaml():
+    """ https://stackoverflow.com/a/8661021 """
+    represent_dict_order = lambda self, data:  self.represent_mapping('tag:yaml.org,2002:map', data.items())
+    yaml.add_representer(OrderedDict, represent_dict_order)    
+setup_yaml()
 
 class AkvoYamlNode(yaml.YAMLObject):
     yaml_tag = u'!AkvoData'
     def __init__(self):
         self.Akvo_VERSION = version
-        self.Import = {}
+        self.Import = OrderedDict() # {}
         self.Processing = OrderedDict() 
-        #self.ProcessingFlow = []
-        #self.Data = {}
-    # For going the other way, data import based on Yaml serialization, 
+    #def __init__(self, node):
+    #    self.Akvo_VERSION = node["version"]
+    #    self.Import = OrderedDict( node["Import"] ) # {}
+    #    self.Processing = OrderedDict( node["Processing"] ) 
     def __repr__(self):
         return "%s(name=%r, Akvo_VESION=%r, Import=%r, Processing=%r)" % (
-            self.__class__.__name__, self.Akvo_VERSION, self.Import, OrderedDict(dict(self.Processing)) ) 
+            #self.__class__.__name__, self.Akvo_VERSION, self.Import, self.Processing )
+            self.__class__.__name__, self.Akvo_VERSION, self.Import, OrderedDict(self.Processing) ) 
+            #self.__class__.__name__, self.Akvo_VERSION, self.Import, OrderedDict(self.Processing)) 
     
 try:    
     import thread 
@@ -563,8 +573,9 @@ class ApplicationWindow(QtWidgets.QMainWindow):
             # Window centres 
 
         with open(SaveStr, 'w') as outfile:
-            for line in self.logText:
-                outfile.write(line+"\n")
+            #for line in self.logText:
+            #    outfile.write(line+"\n")
+            yaml.dump(self.YamlNode, outfile)   
             yaml.dump(INFO, outfile, default_flow_style=False)   
  
     def SavePreprocess(self):
@@ -604,8 +615,6 @@ class ApplicationWindow(QtWidgets.QMainWindow):
         INFO["transFreq"] = self.RAWDataProc.transFreq
         INFO["headerstr"] = str(self.headerstr)
         INFO["log"] = yaml.dump( self.YamlNode )  #self.logText  #MAK 20170127
-        
-        print ("YAML NODE", yaml.dump( self.YamlNode ) )
     
         self.RAWDataProc.DATADICT["INFO"] = INFO 
 
@@ -670,7 +679,15 @@ class ApplicationWindow(QtWidgets.QMainWindow):
             #print ( self.RAWDataProc.DATADICT["INFO"]["log"] )
         
         self.logText = self.RAWDataProc.DATADICT["INFO"]["log"] # YAML 
-        self.YamlNode = yaml.load( self.logText ) 
+
+        self.YamlNode = AkvoYamlNode( )  #self.logText )
+        self.YamlNode.Akvo_VERSION = (yaml.load( self.logText )).Akvo_VERSION
+        self.YamlNode.Import = OrderedDict((yaml.load( self.logText )).Import)
+        self.YamlNode.Processing = OrderedDict((yaml.load( self.logText )).Processing)
+        #self.YamlNode.Akvo_VERSION =  2 #yaml.load( self.logText )["Akvo_VERSION"] #, Loader=yaml.RoundTripLoader) # offending line! 
+        #self.YamlNode = AkvoYamlNode( self.logText ) # offending line! 
+        #print("import type", type( self.YamlNode.Processing ))
+        self.Log() 
             #self.ui.logTextBrowser.append( yaml.dump(self.YamlNode)) #, default_flow_style=False)  )
         #except KeyError:
         #    pass
@@ -848,8 +865,7 @@ class ApplicationWindow(QtWidgets.QMainWindow):
         #for line in nlogText: 
         #    self.ui.logTextBrowser.append( line )
         #    self.logText.append( line ) 
-            
-        self.ui.logTextBrowser.clear() 
+        self.ui.logTextBrowser.clear()
         self.ui.logTextBrowser.append( yaml.dump(self.YamlNode)) #, default_flow_style=False)  )
 
     def disable(self):
@@ -954,7 +970,9 @@ class ApplicationWindow(QtWidgets.QMainWindow):
 
     def calcQ(self):
         if "Calc Q" not in self.YamlNode.Processing.keys():
+            print("In CalcQ", yaml.dump(self.YamlNode.Processing)  )
             self.YamlNode.Processing["Calc Q"] = True
+            print( yaml.dump(self.YamlNode.Processing)  )
             self.Log()
         else:
             err_msg = "Q values have already been calculated"

akvo/tressel/rotate.py

@@ -133,6 +133,11 @@ def RotateAmplitude(X, Y, zeta, df, t):
 
 def gateIntegrate(T2D, T2T, gpd, sigma, stackEfficiency=2.):
     """ Gate integrate the signal to gpd, gates per decade
+        T2D = the time series to gate integrate, complex 
+        T2T = the abscissa values 
+        gpd = gates per decade 
+        sigma = estimate of standard deviation for theoretical gate noise 
+        stackEfficiency = exponential in theoretical gate noise, 2 represents ideal stacking
     """
     
     # use artificial time gates so that early times are fully captured
@@ -140,48 +145,44 @@ def gateIntegrate(T2D, T2T, gpd, sigma, stackEfficiency=2.):
     T2TD = T2T[0] - (T2T[1]-T2T[0])
     T2T -= T2TD
     
-    # calculate total number of decades
-    nd = np.log10(T2T[-1]/T2T[0]) #np.log10(self.T2T[-1]) -  np.log10(self.T2T[-1])
+    #####################################
+    # calculate total number of decades #
+    # windows edges are approximate until binning but will be adjusted to reflect data timing, this 
+    # primarily impacts bins with a few samples  
+    nd = np.log10(T2T[-1]/T2T[0])               
     tdd = np.logspace( np.log10(T2T[0]), np.log10(T2T[-1]), (int)(gpd*nd)+1, base=10, endpoint=True) 
-    tdl = tdd[0:-1]     # these are left edges
-    tdr = tdd[1::]      # these are left edges
-    td = (tdl+tdr) / 2. # window centres
+    tdl = tdd[0:-1]                 # approximate window left edges
+    tdr = tdd[1::]                  # approximate window right edges
+    td = (tdl+tdr) / 2.             # approximate window centres
 
-    Vars = np.ones( len(td) ) * sigma**2 #* .15**2
+
+    Vars = np.zeros( len(td) ) 
     htd = np.zeros( len(td), dtype=complex )
-    isum = np.zeros( len(td) )
-    ii = 0
+    isum = np.zeros( len(td), dtype=int )  
 
-    SIGSTACK = {}
-    SIGSTACK[ii]= []
+    ii = 0
     for itd in range(len(T2T)):
         if ( T2T[itd] > tdr[ii] ):
-            if (ii < len(td)-1):
-                ii += 1
-                SIGSTACK[ii] = []
-            else:
-                #print "overshoot??", ii
-                break
+            ii += 1
+            # correct window edges to centre about data 
+            tdr[ii-1] = (T2T[itd-1]+T2T[itd])*.5 
+            tdl[ii  ] = (T2T[itd-1]+T2T[itd])*.5
         isum[ii] += 1
         htd[ii] += T2D[ itd ]
-        SIGSTACK[ii].append(T2D[itd])
         Vars[ii] += sigma**2
+        
+    td = (tdl+tdr) / 2.             # actual window centres
+    sigma2 = np.sqrt( Vars * ((1/(isum))**stackEfficiency) ) 
 
-    sigma = np.sqrt( Vars  * (1/isum)**stackEfficiency ) 
-    for ii in range(len(td)):
-        if len(SIGSTACK[ii]) > 30:
-            sigma[ii] = np.var(SIGSTACK[ii]) / ((len(SIGSTACK[ii])-1)**(1/stackEfficiency))
-            #sigma[ii] = np.std(SIGSTACK[ii]) * ( 1./(len(SIGSTACK[ii]))**stackEfficiency)
+    # Reset abscissa where isum == 1 
+    # when there is no windowing going on 
+    td[isum==1] = T2T[0:len(td)][isum==1]
 
-    # RESET times where isum == 1
-    ii = 0
-    while (isum[ii] == 1):
-        td[ii] = T2T[ii]
-        ii += 1
+    tdd = np.append(tdl, tdr[-1])
 
-    htd /= isum
-    T2T += T2TD  
-    return td+T2TD, htd, tdd+T2TD, sigma, isum  # centre abscissa, data, window edges, error  
+    htd /= isum # average
+    T2T += T2TD 
+    return td+T2TD, htd, tdd+T2TD, sigma2, isum  # centre abscissa, data, window edges, error 
 
 if __name__ == "__main__":