Akvo changes for kernel calculation (#1)

* Work towards loop info

* improvements, table is getting smarter about loop transmission

* better loop logic

* currently broken, but refactoring to enable processing of legacy data

* Binary file read is restored, working on ASCII

* Legacy FID's can be loaded

* Added option for python phasing rather than R

* Logging improvements with saved files

* A few tweaks

* reduced search for nls

akvo/gui/akvoGUI.py

         return "np.array(%r)" % (self.data)
 
 from collections import OrderedDict
-def represent_ordereddict(dumper, data):
-    value = []
-    for item_key, item_value in data.items():
-        node_key = dumper.represent_data(item_key)
-        node_value = dumper.represent_data(item_value)
-        value.append((node_key, node_value))
-    return yaml.nodes.MappingNode(u'tag:yaml.org,2002:map', value)
-yaml.add_representer(OrderedDict, represent_ordereddict)
+#def represent_ordereddict(dumper, data):
+#    print("representing IN DA HOUSE!!!!!!!!!!!!!!!!!!!!!")
+#    value = []
+#    for item_key, item_value in data.items():
+#        node_key = dumper.represent_data(item_key)
+#        node_value = dumper.represent_data(item_value)
+#        value.append((node_key, node_value))
+#    return yaml.nodes.MappingNode(u'tag:yaml.org,2002:map', value)
+#yaml.add_representer(OrderedDict, represent_ordereddict)
+
+def setup_yaml():
+    """ https://stackoverflow.com/a/8661021 """
+    represent_dict_order = lambda self, data:  self.represent_mapping('tag:yaml.org,2002:map', data.items())
+    yaml.add_representer(OrderedDict, represent_dict_order)    
+setup_yaml()
 
 class AkvoYamlNode(yaml.YAMLObject):
-    yaml_tag = u'!AkvoData'
+    yaml_tag = u'AkvoData'
     def __init__(self):
         self.Akvo_VERSION = version
-        self.Import = {}
+        self.Import = OrderedDict() # {}
         self.Processing = OrderedDict() 
-        #self.ProcessingFlow = []
-        #self.Data = {}
-    # For going the other way, data import based on Yaml serialization, 
+    #def __init__(self, node):
+    #    self.Akvo_VERSION = node["version"]
+    #    self.Import = OrderedDict( node["Import"] ) # {}
+    #    self.Processing = OrderedDict( node["Processing"] ) 
     def __repr__(self):
         return "%s(name=%r, Akvo_VESION=%r, Import=%r, Processing=%r)" % (
-            self.__class__.__name__, self.Akvo_VERSION, self.Import, OrderedDict(dict(self.Processing)) ) 
+            #self.__class__.__name__, self.Akvo_VERSION, self.Import, self.Processing )
+            self.__class__.__name__, self.Akvo_VERSION, self.Import, OrderedDict(self.Processing) ) 
+            #self.__class__.__name__, self.Akvo_VERSION, self.Import, OrderedDict(self.Processing)) 
     
 try:    
     import thread 
         self.ui.mplwidget_navigator_2.setCanvas(self.ui.mplwidget_2)
 
         ##########################################################################
-        # modelling Table 
-        self.ui.loopTableWidget.setRowCount(40)       
-        self.ui.loopTableWidget.setColumnCount(5)      
-        self.ui.loopTableWidget.setHorizontalHeaderLabels( ["ch. tag", "Northing [m]","Easting [m]","Height [m]", "Radius"] )
-        self.ui.loopTableWidget.cellChanged.connect(self.cellChanged) 
+        # Loop Table 
+        self.ui.loopTableWidget.setRowCount(80)       
+        self.ui.loopTableWidget.setColumnCount(6)      
+        self.ui.loopTableWidget.setHorizontalHeaderLabels( ["ch. tag", \
+            "Northing [m]","Easting [m]","Height [m]", "Radius","Tx"] )
+        
+        for ir in range(0, self.ui.loopTableWidget.rowCount() ):
+            for ic in range(1, self.ui.loopTableWidget.columnCount() ):
+                pCell = QtWidgets.QTableWidgetItem()
+                #pCell.setFlags(QtCore.Qt.ItemIsEnabled | QtCore.Qt.ItemIsSelectable)
+                pCell.setFlags(QtCore.Qt.NoItemFlags) # not selectable 
+                pCell.setBackground( QtGui.QColor("lightgrey").lighter(110) )
+                self.ui.loopTableWidget.setItem(ir, ic, pCell)
+
+        self.ui.loopTableWidget.cellChanged.connect(self.loopCellChanged)
+        #self.ui.loopTableWidget.cellPressed.connect(self.loopCellChanged) 
+        self.ui.loopTableWidget.itemClicked.connect(self.loopCellClicked) 
+        #self.ui.loopTableWidget.cellPressed.connect(self.loopCellClicked) 
 
         self.ui.loopTableWidget.setDragDropOverwriteMode(False)
-        self.ui.loopTableWidget.setDragEnabled(True)
-        self.ui.loopTableWidget.setDragDropMode(QtWidgets.QAbstractItemView.InternalMove)
-        
+        self.ui.loopTableWidget.setDragEnabled(False)
+        #self.ui.loopTableWidget.setDragDropMode(QtWidgets.QAbstractItemView.InternalMove)
+
+        self.loops = {}        
+
         ##########################################################################
         # layer Table 
         self.ui.layerTableWidget.setRowCount(80)       
         self.ui.layerTableWidget.setColumnCount(3)      
         self.ui.layerTableWidget.setHorizontalHeaderLabels( [r"top [m]", r"bottom [m]", "σ [ Ωm]" ] )
 
-
+        # do we want this
         self.ui.layerTableWidget.setDragDropOverwriteMode(False)
         self.ui.layerTableWidget.setDragEnabled(True)
         self.ui.layerTableWidget.setDragDropMode(QtWidgets.QAbstractItemView.InternalMove)
         
         pCell = QtWidgets.QTableWidgetItem()
-        #pCell.setFlags(QtCore.Qt.ItemIsEnabled | QtCore.Qt.ItemIsSelectable)
         pCell.setFlags(QtCore.Qt.NoItemFlags) # not selectable 
-        #pCell.setBackground( QtGui.QColor("lightblue").lighter(125) )
         pCell.setBackground( QtGui.QColor("lightgrey").lighter(110) )
         self.ui.layerTableWidget.setItem(0, 1, pCell)
-        for ir in range(1, 80):
-            for ic in range(0, 3):
+        for ir in range(1, self.ui.layerTableWidget.rowCount() ):
+            for ic in range(0, self.ui.layerTableWidget.columnCount() ):
                 pCell = QtWidgets.QTableWidgetItem()
                 #pCell.setFlags(QtCore.Qt.ItemIsEnabled | QtCore.Qt.ItemIsSelectable)
                 pCell.setFlags(QtCore.Qt.NoItemFlags) # not selectable 
                 self.ui.layerTableWidget.setItem(ir, ic, pCell)
         self.ui.layerTableWidget.cellChanged.connect(self.sigmaCellChanged) 
 
+
     def sigmaCellChanged(self):
         self.ui.layerTableWidget.cellChanged.disconnect(self.sigmaCellChanged) 
         # TODO consider building the model whenever this is called. Would be nice to be able to 
                     self.ui.layerTableWidget.cellChanged.connect(self.sigmaCellChanged) 
                     return
 
-            print("I'm here joey")
             # enable next layer
             pCell4 = self.ui.layerTableWidget.item(ii+1, jj)
             pCell4.setBackground( QtGui.QColor("lightblue") ) #.lighter(110))
 
         self.ui.layerTableWidget.cellChanged.connect(self.sigmaCellChanged) 
 
+    def loopCellClicked(self, item):
+        print("checkstate", item.checkState(),item.row())
 
-    def cellChanged(self):
-        # TODO consider building the model whenever this is called. Would be nice to be able to 
-        # do that. Would require instead dist of T2 I guess. 
+        #self.ui.loopTableWidget.itemClicked.disconnect(self.loopCellClicked)
+        jj = item.column() 
+        ii = item.row()
+        tp = type(self.ui.loopTableWidget.item(ii, 0))
+        print("tp", tp, ii, jj)
+        if str(tp) == "<class 'NoneType'>": 
+            return 
+        #print("Clicked", ii, jj)
+        if jj == 5 and self.ui.loopTableWidget.item(ii, 0).text() in self.loops.keys():
+            #print("jj=5")
+            self.loops[ self.ui.loopTableWidget.item(ii, 0).text() ]["Tx"] = self.ui.loopTableWidget.item(ii, 5).checkState()
+            # update surrogates  
+            print("updating surrogates")
+            for point in self.loops[ self.ui.loopTableWidget.item(ii, 0).text() ]["points"][1:]:
+                pCell = self.ui.loopTableWidget.item(point, 5)
+                if self.ui.loopTableWidget.item(ii, 5).checkState():
+                    pCell.setCheckState(QtCore.Qt.Checked);
+                else:
+                    pCell.setCheckState(QtCore.Qt.Unchecked);
+ 
+            #print( "loops",  self.loops[ self.ui.loopTableWidget.item(ii, 0).text() ]["Tx"]) 
+         
+        #self.ui.loopTableWidget.itemClicked.connect(self.loopCellClicked) 
+
+    def loopCellChanged(self):
+        self.ui.loopTableWidget.cellChanged.disconnect(self.loopCellChanged) 
+        
         jj = self.ui.loopTableWidget.currentColumn()
         ii = self.ui.loopTableWidget.currentRow()
-            
-        #if self.ui.loopTableWidget.item(ii, jj) == None:
-        #    return
 
-        try:
-            eval (str( self.ui.loopTableWidget.item(ii, jj).text() ))
-        except:
-            if jj != 0:
-                Error = QtWidgets.QMessageBox()
-                Error.setWindowTitle("Error!")
-                Error.setText("Non-numeric value encountered")
-                Error.setDetailedText("Modelling parameters must be able to be cast into numeric values.")
-                Error.exec_()
+        if jj == 0 and len( self.ui.loopTableWidget.item(ii, jj).text().strip()) == 0:
+            for jjj in range(jj+1,jj+6): 
+                pCell = self.ui.loopTableWidget.item(ii, jjj)
+                pCell.setBackground( QtGui.QColor("white") )
+                pCell.setFlags( QtCore.Qt.NoItemFlags | QtCore.Qt.ItemIsUserCheckable   ) # not selectable 
+        elif jj == 0 and len( self.ui.loopTableWidget.item(ii, jj).text().strip() ): # ch. tag modified
+            for jjj in range(jj+1,jj+5): 
+                pCell = self.ui.loopTableWidget.item(ii, jjj)
+                pCell.setFlags( QtCore.Qt.ItemIsSelectable | QtCore.Qt.ItemIsEditable | QtCore.Qt.ItemIsEnabled )
+                pCell.setBackground( QtGui.QColor("lightblue") )
+            if self.ui.loopTableWidget.item(ii, jj).text() not in self.loops.keys():
+                # This is a new loop ID 
+                self.loops[ self.ui.loopTableWidget.item(ii, jj).text() ] = {}
+                self.loops[ self.ui.loopTableWidget.item(ii, jj).text() ]["Tx"] = self.ui.loopTableWidget.item(ii, 5).checkState()
+                self.loops[ self.ui.loopTableWidget.item(ii, jj).text() ]["points"] = [ii] 
+                # Transmitter cell 
+                pCell = self.ui.loopTableWidget.item(ii, jj+5)
+                pCell.setCheckState(QtCore.Qt.Unchecked)
+                pCell.setFlags( QtCore.Qt.ItemIsUserCheckable | QtCore.Qt.ItemIsEnabled  )
+                pCell.setBackground( QtGui.QColor("lightblue") ) 
+            else:
+                # This is an existing loop ID 
+                self.loops[ self.ui.loopTableWidget.item(ii, jj).text() ]["points"].append( ii ) 
+                pCell = self.ui.loopTableWidget.item(ii, jj+5)
+                pCell.setFlags(QtCore.Qt.NoItemFlags) # not selectable 
+                if self.loops[ self.ui.loopTableWidget.item(ii, 0).text() ]["Tx"]:
+                    pCell.setCheckState(QtCore.Qt.Checked);
+                else:
+                    pCell.setCheckState(QtCore.Qt.Unchecked);
+                #pCell.setFlags( )
+                pCell.setBackground( QtGui.QColor("lightblue") )
+
+
             
+ 
         self.plotLoops()
+        self.ui.loopTableWidget.cellChanged.connect(self.loopCellChanged) 
+
 
     def plotLoops(self):
                
         for ii in range( self.ui.loopTableWidget.rowCount() ):
             for jj in range( self.ui.loopTableWidget.columnCount() ):
                 tp = type(self.ui.loopTableWidget.item(ii, jj))
-                if str(tp) == "<class 'NoneType'>":  # ugly hack needed by PySide for some strange reason.
-                    pass #print ("NONE")
+                if str(tp) == "<class 'NoneType'>":  
+                    pass 
+                elif not len(self.ui.loopTableWidget.item(ii, jj).text()): 
+                    pass
                 else:
                     if jj == 0: 
                         idx = self.ui.loopTableWidget.item(ii, 0).text()
                 self.ui.mplwidget_3.ax1.plot(  np.array(nor[ii]), np.array(eas[ii])  )
             except:
                 pass 
+        #self.ui.mplwidget_3.figure.axes().set
+        plt.gca().set_aspect('equal') #, adjustable='box')
         self.ui.mplwidget_3.draw()
 
     def about(self):
         self.RAWDataProc.updateProcTrigger.connect(self.updateProc)
 
     def openGMRRAWDataset(self):
-
+        """ Opens a GMR header file
+        """
         try:
             with open('.gmr.last.path') as f: 
                 fpath = f.readline()  
             # Window centres 
 
         with open(SaveStr, 'w') as outfile:
-            for line in self.logText:
-                outfile.write(line+"\n")
+            #for line in self.logText:
+            #    outfile.write(line+"\n")
+            yaml.dump(self.YamlNode, outfile)   
             yaml.dump(INFO, outfile, default_flow_style=False)   
  
     def SavePreprocess(self):
         INFO["transFreq"] = self.RAWDataProc.transFreq
         INFO["headerstr"] = str(self.headerstr)
         INFO["log"] = yaml.dump( self.YamlNode )  #self.logText  #MAK 20170127
-        
-        print ("YAML NODE", yaml.dump( self.YamlNode ) )
     
         self.RAWDataProc.DATADICT["INFO"] = INFO 
 
 
     # Export XML file suitable for USGS ScienceBase Data Release
     def ExportXML(self):
-
+        """ This is a filler function for use by USGS collaborators 
+        """
         return 42
 
     def OpenPreprocess(self):
             #print ( self.RAWDataProc.DATADICT["INFO"]["log"] )
         
         self.logText = self.RAWDataProc.DATADICT["INFO"]["log"] # YAML 
-        self.YamlNode = yaml.load( self.logText ) 
+
+        self.YamlNode = AkvoYamlNode( )  #self.logText )
+        self.YamlNode.Akvo_VERSION = (yaml.load( self.logText )).Akvo_VERSION
+        self.YamlNode.Import = OrderedDict((yaml.load( self.logText )).Import)
+        self.YamlNode.Processing = OrderedDict((yaml.load( self.logText )).Processing)
+        #self.YamlNode.Akvo_VERSION =  2 #yaml.load( self.logText )["Akvo_VERSION"] #, Loader=yaml.RoundTripLoader) # offending line! 
+        #self.YamlNode = AkvoYamlNode( self.logText ) # offending line! 
+        #print("import type", type( self.YamlNode.Processing ))
+        self.Log() 
             #self.ui.logTextBrowser.append( yaml.dump(self.YamlNode)) #, default_flow_style=False)  )
         #except KeyError:
         #    pass
         #for line in nlogText: 
         #    self.ui.logTextBrowser.append( line )
         #    self.logText.append( line ) 
-            
-        self.ui.logTextBrowser.clear() 
+        self.ui.logTextBrowser.clear()
         self.ui.logTextBrowser.append( yaml.dump(self.YamlNode)) #, default_flow_style=False)  )
 
     def disable(self):
 
     def calcQ(self):
         if "Calc Q" not in self.YamlNode.Processing.keys():
+            print("In CalcQ", yaml.dump(self.YamlNode.Processing)  )
             self.YamlNode.Processing["Calc Q"] = True
+            print( yaml.dump(self.YamlNode.Processing)  )
             self.Log()
         else:
             err_msg = "Q values have already been calculated"

akvo/gui/main.ui

         <rect>
          <x>0</x>
          <y>0</y>
-         <width>982</width>
-         <height>921</height>
+         <width>967</width>
+         <height>922</height>
         </rect>
        </property>
        <layout class="QHBoxLayout" name="horizontalLayout_2">
            <enum>Qt::LeftToRight</enum>
           </property>
           <property name="currentIndex">
-           <number>0</number>
+           <number>2</number>
           </property>
           <widget class="QWidget" name="tab">
            <property name="minimumSize">
               <string>&lt;!DOCTYPE HTML PUBLIC &quot;-//W3C//DTD HTML 4.0//EN&quot; &quot;http://www.w3.org/TR/REC-html40/strict.dtd&quot;&gt;
 &lt;html&gt;&lt;head&gt;&lt;meta name=&quot;qrichtext&quot; content=&quot;1&quot; /&gt;&lt;style type=&quot;text/css&quot;&gt;
 p, li { white-space: pre-wrap; }
-&lt;/style&gt;&lt;/head&gt;&lt;body style=&quot; font-family:'Sans Serif'; font-size:8pt; font-weight:400; font-style:italic;&quot;&gt;
+&lt;/style&gt;&lt;/head&gt;&lt;body style=&quot; font-family:'Ubuntu'; font-size:8pt; font-weight:400; font-style:italic;&quot;&gt;
 &lt;p style=&quot; margin-top:0px; margin-bottom:0px; margin-left:0px; margin-right:0px; -qt-block-indent:0; text-indent:0px;&quot;&gt;&lt;span style=&quot; font-family:'DejaVu Serif'; font-size:9pt;&quot;&gt;Load supported RAW Dataset header from file menu&lt;/span&gt;&lt;/p&gt;&lt;/body&gt;&lt;/html&gt;</string>
              </property>
             </widget>
               <string>&lt;!DOCTYPE HTML PUBLIC &quot;-//W3C//DTD HTML 4.0//EN&quot; &quot;http://www.w3.org/TR/REC-html40/strict.dtd&quot;&gt;
 &lt;html&gt;&lt;head&gt;&lt;meta name=&quot;qrichtext&quot; content=&quot;1&quot; /&gt;&lt;style type=&quot;text/css&quot;&gt;
 p, li { white-space: pre-wrap; }
-&lt;/style&gt;&lt;/head&gt;&lt;body style=&quot; font-family:'Sans Serif'; font-size:9pt; font-weight:400; font-style:italic;&quot;&gt;
+&lt;/style&gt;&lt;/head&gt;&lt;body style=&quot; font-family:'Ubuntu'; font-size:11pt; font-weight:400; font-style:italic;&quot;&gt;
 &lt;p style=&quot;-qt-paragraph-type:empty; margin-top:0px; margin-bottom:0px; margin-left:0px; margin-right:0px; -qt-block-indent:0; text-indent:0px; font-family:'DejaVu Serif'; font-size:10pt;&quot;&gt;&lt;br /&gt;&lt;/p&gt;&lt;/body&gt;&lt;/html&gt;</string>
              </property>
             </widget>
                 <rect>
                  <x>20</x>
                  <y>37</y>
-                 <width>111</width>
+                 <width>121</width>
                  <height>16</height>
                 </rect>
                </property>
               <widget class="QLabel" name="label_36">
                <property name="geometry">
                 <rect>
-                 <x>560</x>
-                 <y>40</y>
+                 <x>10</x>
+                 <y>190</y>
                  <width>61</width>
                  <height>16</height>
                 </rect>
               <widget class="QLineEdit" name="locEdit">
                <property name="geometry">
                 <rect>
-                 <x>560</x>
-                 <y>60</y>
-                 <width>351</width>
-                 <height>81</height>
+                 <x>10</x>
+                 <y>210</y>
+                 <width>441</width>
+                 <height>51</height>
                 </rect>
                </property>
               </widget>
-              <widget class="QDoubleSpinBox" name="decSpinBox">
+              <widget class="Line" name="line">
                <property name="geometry">
                 <rect>
-                 <x>420</x>
-                 <y>70</y>
-                 <width>101</width>
-                 <height>31</height>
+                 <x>20</x>
+                 <y>160</y>
+                 <width>371</width>
+                 <height>16</height>
                 </rect>
                </property>
-               <property name="decimals">
-                <number>1</number>
+               <property name="orientation">
+                <enum>Qt::Horizontal</enum>
                </property>
-               <property name="minimum">
-                <double>-90.000000000000000</double>
+              </widget>
+              <widget class="QLabel" name="label_52">
+               <property name="geometry">
+                <rect>
+                 <x>0</x>
+                 <y>260</y>
+                 <width>191</width>
+                 <height>31</height>
+                </rect>
                </property>
-               <property name="maximum">
-                <double>90.000000000000000</double>
+               <property name="text">
+                <string>&lt;html&gt;&lt;head/&gt;&lt;body&gt;&lt;p&gt;Comments and field notes&lt;/p&gt;&lt;/body&gt;&lt;/html&gt;</string>
                </property>
-               <property name="value">
-                <double>0.000000000000000</double>
+              </widget>
+              <widget class="QTextBrowser" name="txtComments">
+               <property name="geometry">
+                <rect>
+                 <x>10</x>
+                 <y>300</y>
+                 <width>441</width>
+                 <height>221</height>
+                </rect>
+               </property>
+               <property name="readOnly">
+                <bool>false</bool>
                </property>
               </widget>
-              <widget class="QDoubleSpinBox" name="incSpinBox">
+              <widget class="QTimeEdit" name="timeEdit">
                <property name="geometry">
                 <rect>
-                 <x>420</x>
-                 <y>30</y>
-                 <width>101</width>
-                 <height>31</height>
+                 <x>150</x>
+                 <y>110</y>
+                 <width>118</width>
+                 <height>29</height>
                 </rect>
                </property>
-               <property name="decimals">
-                <number>1</number>
+               <property name="calendarPopup">
+                <bool>true</bool>
                </property>
-               <property name="minimum">
-                <double>-90.000000000000000</double>
+              </widget>
+              <widget class="QDateEdit" name="dateEdit">
+               <property name="geometry">
+                <rect>
+                 <x>150</x>
+                 <y>70</y>
+                 <width>112</width>
+                 <height>29</height>
+                </rect>
                </property>
-               <property name="maximum">
-                <double>90.000000000000000</double>
+               <property name="calendarPopup">
+                <bool>true</bool>
                </property>
-               <property name="value">
-                <double>45.000000000000000</double>
+              </widget>
+              <widget class="QLabel" name="label_50">
+               <property name="geometry">
+                <rect>
+                 <x>24</x>
+                 <y>117</y>
+                 <width>81</width>
+                 <height>16</height>
+                </rect>
+               </property>
+               <property name="text">
+                <string>Survey time</string>
                </property>
               </widget>
-              <widget class="QLabel" name="label_37">
+              <widget class="QDoubleSpinBox" name="tempSpinBox">
                <property name="geometry">
                 <rect>
-                 <x>300</x>
+                 <x>150</x>
                  <y>30</y>
                  <width>111</width>
-                 <height>20</height>
+                 <height>29</height>
                 </rect>
                </property>
-               <property name="text">
-                <string>B Inclination [°]</string>
+               <property name="value">
+                <double>20.000000000000000</double>
                </property>
               </widget>
-              <widget class="QLabel" name="label_38">
+              <widget class="QTableWidget" name="loopTableWidget">
                <property name="geometry">
                 <rect>
-                 <x>300</x>
-                 <y>75</y>
-                 <width>111</width>
-                 <height>20</height>
+                 <x>10</x>
+                 <y>560</y>
+                 <width>641</width>
+                 <height>291</height>
+                </rect>
+               </property>
+               <property name="toolTip">
+                <string>&lt;html&gt;&lt;head/&gt;&lt;body&gt;&lt;p&gt;This table is used to enter coil geometries the format is as follows: each row specifies a single point on a coil. The first column is the coil index (using the GMR channel is useful), the next three colums specify the point in Northing, Easting, and Elevation. These can either be local coordinates or global ones. The final column specifies the loop radius if it is a circle or figure 8, for non circular or figure 8 loops leave this column blank. For figure-8 loops the coils do not need to be touching (see Irons and Kass, 2017). If a given index has 1 row it will be a circular loop, two rows will be a figure 8, and more than that will be a polygonal representation of the points, linearlly interpolated between them. &lt;/p&gt;&lt;/body&gt;&lt;/html&gt;</string>
+               </property>
+              </widget>
+              <widget class="QLabel" name="label_54">
+               <property name="geometry">
+                <rect>
+                 <x>10</x>
+                 <y>540</y>
+                 <width>91</width>
+                 <height>16</height>
                 </rect>
                </property>
                <property name="text">
-                <string>B Declination [°] </string>
+                <string>Surface loops</string>
+               </property>
+              </widget>
+              <widget class="MyDynamicMplCanvas" name="mplwidget_3" native="true">
+               <property name="geometry">
+                <rect>
+                 <x>460</x>
+                 <y>0</y>
+                 <width>500</width>
+                 <height>500</height>
+                </rect>
+               </property>
+               <property name="sizePolicy">
+                <sizepolicy hsizetype="Fixed" vsizetype="Fixed">
+                 <horstretch>0</horstretch>
+                 <verstretch>0</verstretch>
+                </sizepolicy>
+               </property>
+               <property name="minimumSize">
+                <size>
+                 <width>500</width>
+                 <height>500</height>
+                </size>
                </property>
               </widget>
               <widget class="QDoubleSpinBox" name="intensitySpinBox">
                <property name="geometry">
                 <rect>
-                 <x>420</x>
-                 <y>110</y>
+                 <x>790</x>
+                 <y>675</y>
                  <width>101</width>
                  <height>31</height>
                 </rect>
                 <double>50000.000000000000000</double>
                </property>
               </widget>
-              <widget class="QLabel" name="label_51">
+              <widget class="QLabel" name="label_38">
                <property name="geometry">
                 <rect>
-                 <x>300</x>
-                 <y>115</y>
+                 <x>670</x>
+                 <y>640</y>
                  <width>111</width>
                  <height>20</height>
                 </rect>
                </property>
                <property name="text">
-                <string>B Intensity [nT]</string>
+                <string>B Declination [°] </string>
                </property>
               </widget>
-              <widget class="Line" name="line">
+              <widget class="QLabel" name="label_37">
                <property name="geometry">
                 <rect>
-                 <x>20</x>
-                 <y>160</y>
-                 <width>901</width>
-                 <height>16</height>
+                 <x>670</x>
+                 <y>600</y>
+                 <width>111</width>
+                 <height>20</height>
                 </rect>
                </property>
-               <property name="orientation">
-                <enum>Qt::Horizontal</enum>
+               <property name="text">
+                <string>B Inclination [°]</string>
                </property>
               </widget>
-              <widget class="QLabel" name="label_52">
+              <widget class="QDoubleSpinBox" name="decSpinBox">
                <property name="geometry">
                 <rect>
-                 <x>10</x>
-                 <y>180</y>
-                 <width>191</width>
+                 <x>790</x>
+                 <y>635</y>
+                 <width>101</width>
                  <height>31</height>
                 </rect>
                </property>
-               <property name="text">
-                <string>&lt;html&gt;&lt;head/&gt;&lt;body&gt;&lt;p&gt;Comments and field notes&lt;/p&gt;&lt;/body&gt;&lt;/html&gt;</string>
+               <property name="decimals">
+                <number>1</number>
                </property>
-              </widget>
-              <widget class="QTextBrowser" name="txtComments">
-               <property name="geometry">
-                <rect>
-                 <x>30</x>
-                 <y>220</y>
-                 <width>881</width>
-                 <height>631</height>
-                </rect>
+               <property name="minimum">
+                <double>-90.000000000000000</double>
                </property>
-               <property name="readOnly">
-                <bool>false</bool>
+               <property name="maximum">
+                <double>90.000000000000000</double>
+               </property>
+               <property name="value">
+                <double>0.000000000000000</double>
                </property>
               </widget>
-              <widget class="QTimeEdit" name="timeEdit">
+              <widget class="QDoubleSpinBox" name="incSpinBox">
                <property name="geometry">
                 <rect>
-                 <x>140</x>
-                 <y>110</y>
-                 <width>118</width>
-                 <height>29</height>
+                 <x>790</x>
+                 <y>595</y>
+                 <width>101</width>
+                 <height>31</height>
                 </rect>
                </property>
-               <property name="calendarPopup">
-                <bool>true</bool>
+               <property name="decimals">
+                <number>1</number>
+               </property>
+               <property name="minimum">
+                <double>-90.000000000000000</double>
+               </property>
+               <property name="maximum">
+                <double>90.000000000000000</double>
+               </property>
+               <property name="value">
+                <double>45.000000000000000</double>
                </property>
               </widget>
-              <widget class="QDateEdit" name="dateEdit">
+              <widget class="QLabel" name="label_51">
                <property name="geometry">
                 <rect>
-                 <x>140</x>
-                 <y>70</y>
-                 <width>112</width>
-                 <height>29</height>
+                 <x>670</x>
+                 <y>680</y>
+                 <width>111</width>
+                 <height>20</height>
                 </rect>
                </property>
-               <property name="calendarPopup">
-                <bool>true</bool>
+               <property name="text">
+                <string>B Intensity [nT]</string>
                </property>
               </widget>
-              <widget class="QLabel" name="label_50">
+              <widget class="QLabel" name="label_57">
                <property name="geometry">
                 <rect>
-                 <x>24</x>
-                 <y>117</y>
-                 <width>81</width>
+                 <x>670</x>
+                 <y>560</y>
+                 <width>121</width>
                  <height>16</height>
                 </rect>
                </property>
                <property name="text">
-                <string>Survey time</string>
+                <string>Magnetic field</string>
                </property>
               </widget>
-              <widget class="QDoubleSpinBox" name="tempSpinBox">
+              <widget class="Line" name="line_3">
                <property name="geometry">
                 <rect>
-                 <x>140</x>
-                 <y>30</y>
-                 <width>111</width>
-                 <height>29</height>
+                 <x>670</x>
+                 <y>540</y>
+                 <width>251</width>
+                 <height>20</height>
                 </rect>
                </property>
-               <property name="value">
-                <double>20.000000000000000</double>
+               <property name="orientation">
+                <enum>Qt::Horizontal</enum>
                </property>
               </widget>
              </widget>
            <attribute name="title">
             <string>Kernel calc</string>
            </attribute>
-           <widget class="QTableWidget" name="loopTableWidget">
-            <property name="geometry">
-             <rect>
-              <x>20</x>
-              <y>70</y>
-              <width>441</width>
-              <height>271</height>
-             </rect>
-            </property>
-            <property name="toolTip">
-             <string>&lt;html&gt;&lt;head/&gt;&lt;body&gt;&lt;p&gt;This table is used to enter coil geometries the format is as follows: each row specifies a single point on a coil. The first column is the coil index (using the GMR channel is useful), the next three colums specify the point in Northing, Easting, and Elevation. These can either be local coordinates or global ones. The final column specifies the loop radius if it is a circle or figure 8, for non circular or figure 8 loops leave this column blank. For figure-8 loops the coils do not need to be touching (see Irons and Kass, 2017). If a given index has 1 row it will be a circular loop, two rows will be a figure 8, and more than that will be a polygonal representation of the points, linearlly interpolated between them. &lt;/p&gt;&lt;/body&gt;&lt;/html&gt;</string>
-            </property>
-           </widget>
            <widget class="Line" name="line_2">
             <property name="geometry">
              <rect>
              <enum>Qt::Horizontal</enum>
             </property>
            </widget>
-           <widget class="QLabel" name="label_54">
-            <property name="geometry">
-             <rect>
-              <x>20</x>
-              <y>40</y>
-              <width>91</width>
-              <height>16</height>
-             </rect>
-            </property>
-            <property name="text">
-             <string>Surface loops</string>
-            </property>
-           </widget>
-           <widget class="MyDynamicMplCanvas" name="mplwidget_3" native="true">
+           <widget class="MyDynamicMplCanvas" name="mplwidget_4" native="true">
             <property name="geometry">
              <rect>
               <x>480</x>
              <string>&lt;!DOCTYPE HTML PUBLIC &quot;-//W3C//DTD HTML 4.0//EN&quot; &quot;http://www.w3.org/TR/REC-html40/strict.dtd&quot;&gt;
 &lt;html&gt;&lt;head&gt;&lt;meta name=&quot;qrichtext&quot; content=&quot;1&quot; /&gt;&lt;style type=&quot;text/css&quot;&gt;
 p, li { white-space: pre-wrap; }
-&lt;/style&gt;&lt;/head&gt;&lt;body style=&quot; font-family:'Sans Serif'; font-size:9pt; font-weight:400; font-style:normal;&quot;&gt;
-&lt;p style=&quot; margin-top:0px; margin-bottom:0px; margin-left:0px; margin-right:0px; -qt-block-indent:0; text-indent:0px;&quot;&gt;All processing steps are recorded here for your records&lt;/p&gt;&lt;/body&gt;&lt;/html&gt;</string>
+&lt;/style&gt;&lt;/head&gt;&lt;body style=&quot; font-family:'Ubuntu'; font-size:11pt; font-weight:400; font-style:normal;&quot;&gt;
+&lt;p style=&quot; margin-top:0px; margin-bottom:0px; margin-left:0px; margin-right:0px; -qt-block-indent:0; text-indent:0px;&quot;&gt;&lt;span style=&quot; font-family:'Sans Serif'; font-size:9pt;&quot;&gt;All processing steps are recorded here for your records&lt;/span&gt;&lt;/p&gt;&lt;/body&gt;&lt;/html&gt;</string>
             </property>
            </widget>
            <widget class="QLabel" name="label_53">
      <x>0</x>
      <y>0</y>
      <width>1000</width>
-     <height>19</height>
+     <height>25</height>
     </rect>
    </property>
    <widget class="QMenu" name="menuFile">

akvo/tressel/decay.py

-import numpy, pylab,array #,rpy2
-
+import numpy, array #,rpy2
+from matplotlib import pyplot as plt
+import numpy as np
+from scipy.optimize import least_squares
 from rpy2.robjects.packages import importr
 
 import rpy2.robjects as robjects
 
 #################################################
 # Regress for T2 using rpy2 interface
-def regressCurve(peaks,times,sigma2=None  ,intercept=True):
+def regressCurve(peaks,times,sigma2=1,intercept=True):
 
     # TODO, if regression fails, it might be because there is no exponential
     # term, maybe do a second regression then on a linear model. 
     robjects.globalenv['b1'] = b1
     robjects.globalenv['b2'] = b2
     robjects.globalenv['rT2'] = rT2
-    #robjects.globalenv['sigma2'] = sigma2
+    robjects.globalenv['sigma2'] = sigma2
     value = robjects.FloatVector(peaks)
     times = robjects.FloatVector(numpy.array(times))
     
 #    my_weights = robjects.RVector(value/sigma2)
 #    robjects.globalenv['my_weigts'] = my_weights
 
-    if sigma2 != None:
-        # print ("weighting")
-        #tw = numpy.array(peaks)/sigma2 
-        my_weights = robjects.FloatVector( sigma2 )
-    #else:
-    #    my_weights = robjects.FloatVector(numpy.ones(len(peaks))) 
+#    if sigma2 != 0:
+#        print "weighting"
+#        tw = numpy.array(peaks)/sigma2 
+#        my_weights = robjects.RVector( tw/numpy.max(tw) )
+#    else:
+#        my_weights = robjects.RVector(numpy.ones(len(peaks))) 
 
-        robjects.globalenv['my_weights'] = my_weights
-        #print (my_weights)
-        #print (len(peaks))
+#    robjects.globalenv['my_weights'] = my_weights
     
     if (intercept):
         my_list = robjects.r('list(b1=50, b2=1e2, rT2=0.03)')
     env['times'] = times
     
     # ugly, but I get errors with everything else I've tried
-    #my_weights = robjects.r('rep(1,length(value))')
-    #for ii in range(len(my_weights)):
-    #    my_weights[ii] *= peaks[ii]/sigma2
-
-
+    my_weights = robjects.r('rep(1,length(value))')
+    for ii in range(len(my_weights)):
+        my_weights[ii] *= peaks[ii]/sigma2
     Error = False
     #fit = robjects.r.nls(fmla,start=my_list,control=my_cont,weights=my_weights)
-    if (sigma2 != None):
-        #print("SIGMA 2")
+    if (sigma2 != 1):
+        print("SIGMA 2")
         #fit = robjects.r.tryCatch(robjects.r.suppressWarnings(robjects.r.nls(fmla,start=my_list,control=my_cont,algorithm="port", \
         #                     weights=my_weights)), 'silent=TRUE')
+        fit = robjects.r.tryCatch(robjects.r.nls(fmla,start=my_list,control=my_cont))#, \
+                            # weights=my_weights))
+    else:
         try:
-            fit = robjects.r.tryCatch(    robjects.r.nls(fmla, start=my_list, control=my_cont, weights=my_weights, algorithm="port" , \
-                lower=my_lower,upper=my_upper))
+            fit = robjects.r.tryCatch(robjects.r.nls(fmla,start=my_list,control=my_cont,algorithm="port"))#,lower=my_lower,upper=my_upper))
         except:
             print("regression issue pass")
             Error = True
+    # If failure fall back on zero regression values   
+    if not Error:
+        #Error = fit[3][0]
+        report =  r.summary(fit)
+    b1 = 0
+    b2 = 0 
+    rT2 = 1
+    if (intercept):
+        if not Error:
+            b1  =  r['$'](report,'par')[0]
+            b2  =  r['$'](report,'par')[1]
+            rT2 =  r['$'](report,'par')[2]
+            #print  report
+            #print  r['$'](report,'convergence')
+            #print  r['convergence'] #(report,'convergence')
+            #print  r['$'](report,'par')[13]
+            #print  r['$'](report,'par')[14]
+        else:
+            print("ERROR DETECTED, regressed values set to default")
+            b1 = 1e1
+            b2 = 1e-2
+            rT2 = 1e-2
+            #print r['$'](report,'par')[0]
+            #print r['$'](report,'par')[1]
+            #print r['$'](report,'par')[2]
+        return [b1,b2,rT2] 
+    else:
+        if not Error:
+            rT2 =  r['$'](report,'par')[1]
+            b2  =  r['$'](report,'par')[0]
+        else:
+            print("ERROR DETECTED, regressed values set to default")
+        return [b2, rT2] 
+
+#################################################
+# Regress for T2 using rpy2 interface
+def regressCurve2(peaks,times,sigma2=[None],intercept=True):
+
+    if sigma2[0] != None:
+        my_weights = robjects.FloatVector( sigma2 )
+
+    # TODO, if regression fails, it might be because there is no exponential
+    # term, maybe do a second regression then on a linear model. 
+    b1  = 0                  # Bias
+    b2  = 0                  # Linear 
+    bb2  = 0                 # Linear 
+    rT2 = 0.3                # T2 regressed
+    rrT2 = 1.3               # T2 regressed
+    r   = robjects.r         
+
+    # Variable shared between R and Python
+    robjects.globalenv['b1'] = b1
+    robjects.globalenv['b2'] = b2
+    robjects.globalenv['rT2'] = rT2
+    
+    robjects.globalenv['bb2'] = b2
+    robjects.globalenv['rrT2'] = rT2
+    
+    #robjects.globalenv['sigma2'] = sigma2
+    value = robjects.FloatVector(peaks)
+    times = robjects.FloatVector(numpy.array(times))
+    
+    
+    if (intercept):
+        my_list = robjects.r('list(b1=.50, b2=1e2, rT2=0.03, bb2=1e1, rrT2=1.3)')
+        my_lower = robjects.r('list(b1=0, b2=0, rT2=.005, bb2=0, rrT2=.005 )')
+        my_upper = robjects.r('list(b1=2000, b2=2000, rT2=.700, bb2=2000, rrT2=1.3 )')
+    else:
+        my_list  = robjects.r('list(b2=.5, rT2=0.3,  bb2=.5, rrT2=1.3)')
+        my_lower = robjects.r('list(b2=0,  rT2=.005, bb2=0,  rrT2=.005)')
+        my_upper = robjects.r('list(b2=1,  rT2=2.6,    bb2=1,  rrT2=2.6)')
+
+    my_cont = robjects.r('nls.control(maxiter=1000, warnOnly=TRUE, printEval=FALSE)')
+
+    
+    if (intercept):
+        #fmla = robjects.RFormula('value ~ b1 + exp(-times/rT2)')
+        fmla = robjects.Formula('value ~ b1 + b2*exp(-times/rT2) + bb2*exp(-times/rrT2)')
+        #fmla = robjects.RFormula('value ~ b1 + b2*times + exp(-times/rT2)')
+    else:
+        fmla = robjects.Formula('value ~ b2*exp(-times/rT2) + bb2*exp(-times/rrT2)')
+
+    env = fmla.getenvironment()
+    env['value'] = value
+    env['times'] = times
+    
+    # ugly, but I get errors with everything else I've tried
+    Error = False
+    #fit = robjects.r.nls(fmla,start=my_list,control=my_cont,weights=my_weights)
+    if (sigma2[0] != None):
+        #print("SIGMA 2")
+        #fit = robjects.r.tryCatch(robjects.r.suppressWarnings(robjects.r.nls(fmla,start=my_list,control=my_cont,algorithm="port", \
+        #                     weights=my_weights)), 'silent=TRUE')
+        fit = robjects.r.tryCatch(robjects.r.nls(fmla,start=my_list,control=my_cont,algorithm='port',weights=my_weights,lower=my_lower,upper=my_upper))#, \
                             # weights=my_weights))
     else:
         try:
-            fit = robjects.r.tryCatch(robjects.r.nls(fmla,start=my_list,control=my_cont,algorithm="port",lower=my_lower,upper=my_upper))
+            fit = robjects.r.tryCatch(robjects.r.nls(fmla,start=my_list,control=my_cont,algorithm="port"))#,lower=my_lower,upper=my_upper))
         except:
             print("regression issue pass")
             Error = True
             #print r['$'](report,'par')[0]
             #print r['$'](report,'par')[1]
             #print r['$'](report,'par')[2]
-        return [b1,b2,rT2] 
+        return [b1,b2,rT2, bb2, rrT2] 
     else:
         if not Error:
             rT2 =  r['$'](report,'par')[1]
             b2  =  r['$'](report,'par')[0]
+            rrT2 =  r['$'](report,'par')[3]
+            bb2  =  r['$'](report,'par')[2]
         else:
             print("ERROR DETECTED, regressed values set to default")
-        return [b2, rT2] 
+        return [b2, rT2, bb2, rrT2] 
+
+def fun(x, t, y):
+    """ Cost function for regression, single exponential, no DC term 
+        x[0] = A0
+        x[1] = zeta 
+        x[2] = df
+        x[3] = T2
+    """
+    # concatenated real and imaginary parts  
+    pre =  np.concatenate((-x[0]*np.sin(2.*np.pi*x[2]*t + x[1])*np.exp(-t/x[3]), \
+                           +x[0]*np.cos(2.*np.pi*x[2]*t + x[1])*np.exp(-t/x[3])))  
+    return y-pre
+
+def fun2(x, t, y):
+    """ Cost function for regression, single exponential, no DC term 
+        x[0] = A0
+        x[1] = zeta 
+        x[2] = T2
+    """
+    # concatenated real and imaginary parts  
+    pre =  np.concatenate((x[0]*np.cos(x[1])*np.exp(-t/x[2]), \
+                       -1.*x[0]*np.sin(x[1])*np.exp(-t/x[2])))  
+    return y-pre
+
+
+def quadratureDetect2(X, Y, tt, x0="None"): 
+    """ Pure python quadrature detection using Scipy.  
+        X = real part of NMR signal 
+        Y = imaginary component of NMR signal 
+        tt = time 
+    """
+    print("Pure Python Quad Det")
+    # df 
+    if x0=="None":
+        x0 = np.array( [1., 0., 0., .2] )
+        res_lsq = least_squares(fun, x0, args=(tt, np.concatenate((X, Y))), loss='cauchy', f_scale=1.0,\
+            bounds=( [1., -np.pi, -5, .005] , [1000., np.pi, 5, .800] ))
+    else:
+        res_lsq = least_squares(fun, x0, args=(tt, np.concatenate((X, Y))), loss='cauchy', f_scale=1.0,\
+            bounds=( [1., -np.pi, -5, .005] , [1000., np.pi, 5, .800] ))
 
-def quadratureDetect(X, Y, tt):
-    
-    r   = robjects.r         
+        #bounds=( [0., 0, -20, .0] , [1., np.pi, 20, .6] ))
 
-    robjects.r(''' 
-         Xc <- function(E0, df, tt, phi, T2) {
-	            E0 * -sin(2*pi*df*tt + phi) * exp(-tt/T2)
-                }
+    x = res_lsq.x 
+    return res_lsq.success, x[0], x[2], x[1], x[3]
+    
+    # no df
+    #x = np.array( [1., 0., 0.2] )
+    #res_lsq = least_squares(fun2, x, args=(tt, np.concatenate((X, Y))), loss='soft_l1', f_scale=0.1)
+    #x = res_lsq.x 
+    #return conv, E0,df,phi,T2
+    #return res_lsq.success, x[0], 0, x[1], x[2]
 
-         Yc <- function(E0, df, tt, phi, T2) {
-	            E0 * cos(2*pi*df*tt + phi) * exp(-tt/T2)
-                } 
-         ''')  
+def quadratureDetect(X, Y, tt, CorrectFreq=False, BiExp=False, CorrectDC=False):
+ 
+    r   = robjects.r        
+
+    if CorrectDC:
+        robjects.r(''' 
+             Xc1 <- function(E01, df, tt, phi, T2_1, DC) {
+	                DC + E01*cos(2*pi*df*tt + phi) * exp(-tt/T2_1)
+            }
+    
+            Yc1 <- function(E01, df, tt, phi, T2_1, DC) {
+	                DC - E01*sin(2*pi*df*tt + phi) * exp(-tt/T2_1)
+            } 
+            ''')
+    else:   
+        robjects.r(''' 
+             Xc1 <- function(E01, df, tt, phi, T2_1) {
+	                E01*cos(2*pi*df*tt + phi) * exp(-tt/T2_1)
+            }
+    
+            Yc1 <- function(E01, df, tt, phi, T2_1) {
+	                -E01*sin(2*pi*df*tt + phi) * exp(-tt/T2_1)
+            } 
+            ''')
+
+    # bi-exponential 
+    if CorrectDC:
+        robjects.r(''' 
+             Xc2 <- function(E01, E02, df, tt, phi, T2_1, T2_2, DC) {
+	               DC + E01*cos(2*pi*df*tt + phi) * exp(-tt/T2_1) + 
+	                DC + E02*cos(2*pi*df*tt + phi) * exp(-tt/T2_2)
+            }
+
+            Yc2 <- function(E01, E02, df, tt, phi, T2_1, T2_2, DC) {
+	                DC - E01*sin(2*pi*df*tt + phi) * exp(-tt/T2_1) + 
+	                DC - E02*sin(2*pi*df*tt + phi) * exp(-tt/T2_2)
+            } 
+            ''')
+    else:   
+        robjects.r(''' 
+             Xc2 <- function(E01, E02, df, tt, phi, T2_1, T2_2) {
+	               E01*cos(2*pi*df*tt + phi) * exp(-tt/T2_1) + 
+	               E02*cos(2*pi*df*tt + phi) * exp(-tt/T2_2)
+            }
+
+            Yc2 <- function(E01, E02, df, tt, phi, T2_1, T2_2) {
+	                -E01*sin(2*pi*df*tt + phi) * exp(-tt/T2_1) + 
+	                -E02*sin(2*pi*df*tt + phi) * exp(-tt/T2_2)
+            } 
+            ''')
 
     # Make 0 vector 
     Zero = robjects.FloatVector(numpy.zeros(len(X)))
     
     # Fitted Parameters
-    E0 = 0.
+    E01 = 0.
+    E02 = 0.
     df = 0.
     phi = 0.
-    T2 = 0.
-    robjects.globalenv['E0'] = E0
+    T2_1 = 0.
+    T2_2 = 0.
+    DC = 0.
+    robjects.globalenv['DC'] = DC
+    robjects.globalenv['E01'] = E01
+    robjects.globalenv['E02'] = E02
     robjects.globalenv['df'] = df
     robjects.globalenv['phi'] = phi
-    robjects.globalenv['T2'] = T2
+    robjects.globalenv['T2_1'] = T2_1
+    robjects.globalenv['T2_2'] = T2_2
     XY = robjects.FloatVector(numpy.concatenate((X,Y)))
     
     # Arrays
     Y = robjects.FloatVector(numpy.array(Y))
     Zero = robjects.FloatVector(numpy.array(Zero))
 
-    #fmla = robjects.Formula('Zero ~ QI( E0, df, tt, phi, T2, X, Y )')
-    #fmla = robjects.Formula('X ~ Xc( E0, df, tt, phi, T2 )')
-    #fmla = robjects.Formula('Y ~ Yc( E0, df, tt, phi, T2 )')
-    fmla = robjects.Formula('XY ~ c(Xc( E0, df, tt, phi, T2 ), Yc( E0, df, tt, phi, T2 ))')
+    
+
+    if BiExp:
+        if CorrectDC:
+            fmla = robjects.Formula('XY ~ c(Xc2( E01, E02, df, tt, phi, T2_1, T2_2, DC ), Yc2( E01, E02, df, tt, phi, T2_1, T2_2, DC ))')
+            if CorrectFreq:    
+                start = robjects.r('list(E01=.100, E02=.01,   df=0,    phi=0.    ,  T2_1=.100, T2_2=.01, DC=0.0)')
+                lower = robjects.r('list(E01=1e-6, E02=1e-6,  df=-50,  phi=-3.14 ,  T2_1=.001, T2_2=.001, DC=0.0)')
+                upper = robjects.r('list(E01=1.00, E02=1.0,   df=50,   phi=3.14  ,  T2_1=.800, T2_2=.8, DC=0.5)')
+            else:
+                start = robjects.r('list(E01=.100, E02=.01,   phi=0.9   ,  T2_1=.100, T2_2=.01,  DC=0.0)')
+                lower = robjects.r('list(E01=1e-6, E02=1e-6,  phi=-3.14 ,  T2_1=.001, T2_2=.001, DC=0.0)')
+                upper = robjects.r('list(E01=1.00, E02=1.0,   phi=3.14  ,  T2_1=.800, T2_2=.8,   DC=0.5)')
+        else:
+            fmla = robjects.Formula('XY ~ c(Xc2( E01, E02, df, tt, phi, T2_1, T2_2 ), Yc2( E01, E02, df, tt, phi, T2_1, T2_2))')
+            if CorrectFreq:    
+                start = robjects.r('list(E01=.100, E02=.01,   df=0,    phi=0.    ,  T2_1=.100, T2_2=.01)')
+                lower = robjects.r('list(E01=1e-6, E02=1e-6,  df=-50,  phi=-3.14 ,  T2_1=.001, T2_2=.001)')
+                upper = robjects.r('list(E01=1.00, E02=1.0,   df=50,   phi=3.14  ,  T2_1=.800, T2_2=.8)')
+            else:
+                start = robjects.r('list(E01=.100, E02=.01,   phi=0.9   ,  T2_1=.100, T2_2=.01)')
+                lower = robjects.r('list(E01=1e-6, E02=1e-6,  phi=-3.14 ,  T2_1=.001, T2_2=.001)')
+                upper = robjects.r('list(E01=1.00, E02=1.0,   phi=3.14  ,  T2_1=.800, T2_2=.8)')
+    else: 
+        if CorrectDC:
+            fmla = robjects.Formula('XY ~ c(Xc1( E01, df, tt, phi, T2_1, DC), Yc1( E01, df, tt, phi, T2_1,DC))')
+            if CorrectFreq:    
+                start = robjects.r('list(E01=.100, df=0   , phi=0.   , T2_1=.100, DC=0.0)')
+                lower = robjects.r('list(E01=1e-6, df=-50., phi=-3.14, T2_1=.001, DC=0.0)')
+                upper = robjects.r('list(E01=1.00, df=50. , phi=3.14 , T2_1=.800, DC=0.5)')
+            else:
+                start = robjects.r('list(E01=.100, phi= 0.  , T2_1=.100, DC=0.0)')
+                lower = robjects.r('list(E01=1e-6, phi=-3.13, T2_1=.001, DC=0.0)')
+                upper = robjects.r('list(E01=1.00, phi= 3.13, T2_1=.800, DC=0.5)')
+        else:
+            fmla = robjects.Formula('XY ~ c(Xc1( E01, df, tt, phi, T2_1), Yc1( E01, df, tt, phi, T2_1))')
+            if CorrectFreq:    
+                start = robjects.r('list(E01=.100, df=0     , phi=0.   ,  T2_1=.100)')
+                lower = robjects.r('list(E01=1e-6, df=-50. , phi=-3.14 ,  T2_1=.001)')
+                upper = robjects.r('list(E01=1.00, df=50.  , phi=3.14  ,  T2_1=.800)')
+            else:
+                start = robjects.r('list(E01=.100, phi= 0.  , T2_1=.100)')
+                lower = robjects.r('list(E01=1e-6, phi=-3.13, T2_1=.001)')
+                upper = robjects.r('list(E01=1.00, phi= 3.13, T2_1=.800)')
 
     env = fmla.getenvironment()
     env['Zero'] = Zero
     env['Y'] = Y
     env['XY'] = XY 
     env['tt'] = tt
-    
-    # Bounds and control    
-    start = robjects.r('list(E0=100,   df= 0   , phi=   0.00,  T2=.100)')
-    lower = robjects.r('list(E0=1,     df=-13.0, phi=  -3.14,  T2=.005)')
-    upper = robjects.r('list(E0=1000,  df= 13.0, phi=   3.14,  T2=.800)')
 
     cont = robjects.r('nls.control(maxiter=10000, warnOnly=TRUE, printEval=FALSE)')
     
     fit = robjects.r.tryCatch(robjects.r.nls(fmla, start=start, control=cont, lower=lower, upper=upper, algorithm='port')) #, \
+    #fit = robjects.r.tryCatch(robjects.r.nls(fmla, start=start, control=cont)) #, \
     report =  r.summary(fit)
-    #print (report)
-    
-    E0  =  r['$'](report,'par')[0]
-    df  =  r['$'](report,'par')[1]
-    phi =  r['$'](report,'par')[2]
-    T2  =  r['$'](report,'par')[3]
-    #print ( E0,df,phi,T2 )
-    return E0,df,phi,T2
-    
-
-#################################################
-# Regress for T2 using rpy2 interface
-def regressSpec(w, wL, X): #,sigma2=1,intercept=True):
 
-    # compute s
-    s = -1j*w
-
-    # TODO, if regression fails, it might be because there is no exponential
-    # term, maybe do a second regression then on a linear model. 
-    a   = 0                  # Linear 
-    rT2 = 0.1                # T2 regressed
-    r   = robjects.r         
-
-    # Variable shared between R and Python
-    robjects.globalenv['a'] = a
-    #robjects.globalenv['w'] = w
-    robjects.globalenv['rT2'] = rT2
-    robjects.globalenv['wL'] = wL
-    robjects.globalenv['nb'] = 0
-
-    #s = robjects.ComplexVector(numpy.array(s))
-    w = robjects.FloatVector(numpy.array(w))
-    XX = robjects.FloatVector(X)
-    #Xr = robjects.FloatVector(numpy.real(X))
-    #Xi = robjects.FloatVector(numpy.imag(X))
-    #Xa = robjects.FloatVector(numpy.abs(X))
-    #Xri = robjects.FloatVector(numpy.concatenate((Xr,Xi)))
-    
-    #my_lower = robjects.r('list(a=.001, rT2=.001, nb=.0001)')
-    my_lower = robjects.r('list(a=.001, rT2=.001)')
-    #my_upper = robjects.r('list(a=1.5, rT2=.300, nb =100.)')
-    my_upper = robjects.r('list(a=1.5, rT2=.300)')
-     
-    #my_list = robjects.r('list(a=.2, rT2=0.03, nb=.1)')
-    my_list = robjects.r('list(a=.2, rT2=0.03)')
-    my_cont = robjects.r('nls.control(maxiter=5000, warnOnly=TRUE, printEval=FALSE)')
-    
-    #fmla = robjects.Formula('Xri ~ c(a*Re((wL) / (wL^2+(s+1/rT2)^2 )), a*Im((wL)/(wL^2 + (s+1/rT2)^2 )))') # envelope
-    ##fmla = robjects.Formula('Xri ~ c(a*Re((wL) / (wL^2+(s+1/rT2)^2 )), a*Im((wL)/(wL^2 + (s+1/rT2)^2 )))') # envelope
-    #fmla = robjects.Formula('XX ~ a*(wL) / (wL^2 + (s+1/rT2)^2 )') # complex
-    #fmla = robjects.Formula('Xa ~ abs(a*(wL) / (wL^2 + (s+1/rT2)^2 )) + nb') # complex
-    #fmla = robjects.Formula('XX ~ Re(a*( s + 1./rT2 )  / (wL^2 + (s+1/rT2)^2 ))') # complex
-    fmla = robjects.Formula('XX ~ a*(.5/rT2)  / ((1./rT2)^2 + (w-wL)^2 )')
-    #fmla = robjects.Formula('Xa ~ (s + 1./T2) / ( wL**2 + (1/T2 + 1j*w)**2 ) ')
- 
-    env = fmla.getenvironment()
-    #env['s'] = s
-    env['w'] = w
-    #env['Xr'] = Xr
-    #env['Xa'] = Xa
-    #env['Xi'] = Xi
-    #env['Xri'] = Xri
-    env['XX'] = XX
-     
-    #fit = robjects.r.tryCatch(robjects.r.nls(fmla,start=my_list, control=my_cont)) #, lower=my_lower, algorithm='port')) #, \
-    fit = robjects.r.tryCatch(robjects.r.nls(fmla, start=my_list, control=my_cont, lower=my_lower, upper=my_upper, algorithm='port')) #, \
-    report =  r.summary(fit)
-    #print report 
-    #print(r.warnings())
- 
-    a  =  r['$'](report,'par')[0]
-    rT2 =  r['$'](report,'par')[1]
-    nb =  r['$'](report,'par')[2]
+    conv = r['$'](fit,'convergence')[0]
+    #if conv:
+    #    print (report)
+    #    print ("conv", conv)
+    print ("Conv",  r['$'](fit,'convergence'))  # T2
+    print (report)
+    
+    if BiExp:
+        if CorrectFreq:    
+            E0   =  r['$'](report,'par')[0]   # E01
+            E0  +=  r['$'](report,'par')[1]   # E02
+            df  =  r['$'](report,'par')[2]   # offset
+            phi =  r['$'](report,'par')[3]   # phase 
+            T2  =  r['$'](report,'par')[4]   # T2
+        else:
+            E0   =  r['$'](report,'par')[0]   # E01
+            E0  +=  r['$'](report,'par')[1]   # E02
+            phi =  r['$'](report,'par')[2]   # phase 
+            T2  =  r['$'](report,'par')[3]   # T2
+    else:
+        if CorrectFreq:    
+            E0   =  r['$'](report,'par')[0]   # E01
+            df  =  r['$'](report,'par')[1]   # offset
+            phi =  r['$'](report,'par')[2]   # phase 
+            T2  =  r['$'](report,'par')[3]   # T2
+        else:
+            E0   =  r['$'](report,'par')[0]   # E01
+            phi =  r['$'](report,'par')[1]   # phase 
+            T2  =  r['$'](report,'par')[2]   # T2
+    #phi = 0.907655876627
+    #phi = 0
+    #print ("df", df)# = 0
+    return conv, E0,df,phi,T2
     
-    return a, rT2, nb
 
 #################################################
 # Regress for T2 using rpy2 interface
-def regressModulus(w, wL, X): #,sigma2=1,intercept=True):
+def regressSpec(w, wL, X): #,sigma2=1,intercept=True):
 
     # compute s
     s = -1j*w
     rT2 =  r['$'](report,'par')[1]
     nb =  r['$'](report,'par')[2]
     
-    return a, rT2
+    return a, rT2, nb
 
 #################################################
 # Regress for T2 using rpy2 interface
-def regressSpecComplex(w, wL, X, known=True, win=None): #,sigma2=1,intercept=True):
+def regressSpecComplex(w, wL, X): #,sigma2=1,intercept=True):
 
     # compute s
     s = -1j*w
      
     #print (numpy.shape(X))
     
-    #######################################################################
-
-    if known:
-        # known Frequency
-        my_lower = robjects.r('list(a=.001, rT2=.001, phi2=-3.14)')
-        my_upper = robjects.r('list(a=3.5, rT2=.300, phi2=3.14)')
-        my_list = robjects.r('list(a=.2, rT2=0.03, phi2=0)')
-    else:
-        # Unknown Frequency
-        my_lower = robjects.r('list(a=.001, rT2=.001, phi2=-3.14, wL2=wL-5)')
-        my_upper = robjects.r('list(a=3.5, rT2=.300, phi2=3.14, wL2=wL+5)')
-        my_list = robjects.r('list(a=.2, rT2=0.03, phi2=0, wL2=wL)')
-    
+    #my_lower = robjects.r('list(a=.001, rT2=.001, nb=.0001)')
+    #my_lower = robjects.r('list(a=.001, rT2=.001)') # Working
+    my_lower = robjects.r('list(a=.001, rT2=.001, phi2=-3.14, wL2=wL-5)')
+    #my_upper = robjects.r('list(a=1.5, rT2=.300, nb =100.)')
+    my_upper = robjects.r('list(a=3.5, rT2=.300, phi2=3.14, wL2=wL+5)')
+     
+    #my_list = robjects.r('list(a=.2, rT2=0.03, nb=.1)')
+    my_list = robjects.r('list(a=.2, rT2=0.03, phi2=0, wL2=wL)')
     my_cont = robjects.r('nls.control(maxiter=5000, warnOnly=TRUE, printEval=FALSE)')
     
     #fmla = robjects.Formula('Xri ~ c(a*Re((wL) / (wL^2+(s+1/rT2)^2 )), a*Im((wL)/(wL^2 + (s+1/rT2)^2 )))') # envelope
     
     #fmlar = robjects.Formula('Xr ~ (Kwr(a, phi2, s, rT2, wL)) ') # envelope
     #fmlai = robjects.Formula('Xi ~ (Kwi(a, phi2, s, rT2, wL)) ') # envelope
-    
-
-    
-    if known:
-        ###########################################3
-        # KNOWN freq 
-        fmla = robjects.Formula('Xri ~ c(Kwr(a, phi2, s, rT2, wL), Kwi(a, phi2, s, rT2, wL) ) ') # envelope
-    else:
-        ####################################################################################################3
-        # unknown frequency
-        fmla = robjects.Formula('Xri ~ c(Kwr(a, phi2, s, rT2, wL2), Kwi(a, phi2, s, rT2, wL2) ) ') # envelope
-
+    fmla = robjects.Formula('Xri ~ c(Kwr(a, phi2, s, rT2, wL2), Kwi(a, phi2, s, rT2, wL2) ) ') # envelope
     #fmla = robjects.Formula('Xri ~ (Kwri(a, phi2, s, rT2, wL)) ') # envelope
     
     #fmla = robjects.Formula('Xa ~ (abs(a*(sin(phi2)*s + ((1/rT2)*sin(phi2)) + wL*cos(phi2)) / (wL^2+(s+1/rT2)^2 )))') # envelope
     env['Xri'] = Xri
     env['XX'] = XX
      
-    #fit = robjects.r.tryCatch(robjects.r.nls(fmla,start=my_list, control=my_cont)) #, lower=my_lower, algorithm='port')) #, \
-    #fit = robjects.r.tryCatch(robjects.r.nls(fmlar, start=my_list, control=my_cont, lower=my_lower, upper=my_upper, algorithm='port')) #, \
-    fit = robjects.r.tryCatch(robjects.r.nls(fmla, start=my_list, control=my_cont, lower=my_lower, upper=my_upper, algorithm='port')) #, \
+    fit = robjects.r.tryCatch(robjects.r.nls(fmla,start=my_list, control=my_cont)) #, lower=my_lower, algorithm='port')) #, \
+    #fitr = robjects.r.tryCatch(robjects.r.nls(fmlar, start=my_list, control=my_cont, lower=my_lower, upper=my_upper, algorithm='port')) #, \
     
     #env = fmlai.getenvironment()
     #fiti = robjects.r.tryCatch(robjects.r.nls(fmlai, start=my_list, control=my_cont, lower=my_lower, upper=my_upper, algorithm='port')) #, \
     #print( reportr )
     #print( reporti  )
     #exit()
-    #print ( r.warnings())
+    #print  r.warnings()
  
     #a   =  (r['$'](reportr,'par')[0] + r['$'](reporti,'par')[0]) / 2.
     #rT2 =  (r['$'](reportr,'par')[1] + r['$'](reporti,'par')[1]) / 2.
     rT2 =  r['$'](report,'par')[1] 
     nb  =  r['$'](report,'par')[2] #phi2 
 
-    #print ("Python wL2", r['$'](report,'par')[3] )   
-    #print ("Python zeta", r['$'](report,'par')[2] )   
+    print ("Python wL2", r['$'](report,'par')[3] )   
+    print ("Python zeta", r['$'](report,'par')[2] )   
  
     return a, rT2, nb
 
     envelope   =  numpy.exp(-t/T2)
     renvelope  =  numpy.exp(-t/rT2)
 
-    outf = file('regress.txt','w')
-    for i in range(len(times)):
-        outf.write(str(times[i]) + "   " +  str(peaks[i]) + "\n")  
-    outf.close()
+    #outf = file('regress.txt','w')
+    #for i in range(len(times)):
+    #    outf.write(str(times[i]) + "   " +  str(peaks[i]) + "\n")  
+    #outf.close()
 
-    pylab.plot(t,data, 'b')
-    pylab.plot(t,cdata, 'g', linewidth=1)
-    pylab.plot(t,envelope, color='violet', linewidth=4)
-    pylab.plot(t,renvelope, 'r', linewidth=4)
-    pylab.plot(times, numpy.array(peaks), 'bo', markersize=8, alpha=.25)
-    pylab.legend(['noisy data','clean data','real envelope','regressed env','picks'])
-    pylab.savefig("regression.pdf")
+    plt.plot(t,data, 'b')
+    plt.plot(t,cdata, 'g', linewidth=1)
+    plt.plot(t,envelope, color='violet', linewidth=4)
+    plt.plot(t,renvelope, 'r', linewidth=4)
+    plt.plot(times, numpy.array(peaks), 'bo', markersize=8, alpha=.25)
+    plt.legend(['noisy data','clean data','real envelope','regressed env','picks'])
+    plt.savefig("regression.pdf")
 
 
     # FFT check
     fourier = fft(data)
-    pylab.figure()
+    plt.figure()
     freq = fftfreq(len(data), d=dt)
-    pylab.plot(freq, (fourier.real))
+    plt.plot(freq, (fourier.real))
     
-    pylab.show()
+    plt.show()
 
     # TODO do a bunch in batch mode to see if T2 estimate is better with or without 
     # weighting and which model is best.

akvo/tressel/mrsurvey.py

 import sys
 import scipy
 import copy
+import struct
 from scipy.io.matlab import mio
 from numpy import pi
 from math import floor
 import matplotlib.ticker 
 from matplotlib.ticker import MaxNLocator
 
+import multiprocessing 
+import itertools 
+
 import akvo.tressel.adapt as adapt
 #import akvo.tressel.cadapt as adapt # cython for more faster
 import akvo.tressel.decay as decay
 plt.register_cmap(name='magma', cmap=cmaps.magma)
 plt.register_cmap(name='magma_r', cmap=cmaps.magma_r)
 
+
+def loadGMRBinaryFID( rawfname, istack, info ):
+    """ Reads a single binary GMR file and fills into DATADICT
+    """
+
+    #################################################################################
+    # figure out key data indices
+    # Pulse        
+    nps  = (int)((info["prePulseDelay"])*info["samp"])
+    npul   = (int)(self.pulseLength[0]*self.samp) #+ 100 
+
+    # Data 
+    nds  = nps+npul+(int)((self.deadTime)*self.samp);        # indice pulse 1 data starts 
+    nd1 = (int)(1.*self.samp)                                # samples in first pulse
+
+    invGain = 1./self.RxGain        
+    invCGain = self.CurrentGain        
+
+    pulse = "Pulse 1"
+    chan = self.DATADICT[pulse]["chan"] 
+    rchan = self.DATADICT[pulse]["rchan"] 
+        
+    rawFile = open( rawfname, 'rb')
+        
+    T = N_samp * self.dt 
+    TIMES = np.arange(0, T, self.dt) - .0002 # small offset in GMR DAQ?
+
+    for ipm in range(self.nPulseMoments):
+        buf1 = rawFile.read(4)
+        buf2 = rawFile.read(4)
+                
+        N_chan = struct.unpack('>i', buf1 )[0]
+        N_samp = struct.unpack('>i', buf2 )[0]
+
+        DATA = np.zeros([N_samp, N_chan+1])
+        for ichan in range(N_chan):
+            DATADUMP = rawFile.read(4*N_samp)
+            for irec in range(N_samp):
+                DATA[irec,ichan] = struct.unpack('>f', DATADUMP[irec*4:irec*4+4])[0]
+        
+    return DATA, TIMES
+
 class SNMRDataProcessor(QObject):
     """ Revised class for preprocessing sNMR Data. 
         Derived types can read GMR files  
         self.transFreq       = HEADER[1]
         self.maxBusV         = HEADER[2]
         self.pulseLength     = pulseLengthDict.get((int)(HEADER[0]))(1e-3*HEADER[3])
-        self.interpulseDelay = 1e-3*HEADER[4]   # for T2, Spin Echo
-        self.repetitionDelay = HEADER[5]        # delay between first pulse
-        self.nPulseMoments   = (int)(HEADER[6]) # Number of pulse moments per stack
-        self.TuneCapacitance = HEADER[7]        # tuning capacitance in uF
-        self.nTransVersion   = HEADER[8]        # Transmitter version
-        self.nDAQVersion     = HEADER[9]        # DAQ software version 
-        self.nInterleaves    = HEADER[10]       # num interleaves
+        self.interpulseDelay = 1e-3*HEADER[4]      # for T2, Spin Echo
+        self.repetitionDelay = HEADER[5]           # delay between first pulse
+        self.nPulseMoments   = (int)(HEADER[6])    # Number of pulse moments per stack
+        self.TuneCapacitance = HEADER[7]           # tuning capacitance in uF
+        self.nTransVersion   = HEADER[8]           # Transmitter version
+        self.nDAQVersion     = HEADER[9]           # DAQ software version 
+        self.nInterleaves    = HEADER[10]          # num interleaves
+
         self.gain()
         
         # default 
 
         # newer header files contain 64 entries
         if self.nDAQVersion >= 2:
-            #print "new file format"
+           #self.deadtime       = HEADER[11]
+           #self.unknown        = HEADER[12]
+           #self.PreAmpGain     = HEADER[13]
             self.samp           = HEADER[14]     # sampling frequency
             self.dt             = 1./self.samp   # sampling rate 
             self.deadTime       = .0055          # instrument dead time before measurement
         # Current gain
         if floor(self.nDAQVersion) == 1:
             self.CurrentGain = 150.
-        elif floor(self.nDAQVersion) ==2:
+        elif floor(self.nDAQVersion) == 2:
             self.CurrentGain = 180.
 
     def updateProgress(self):
         NRmax = {}
         REmax = {}
         IMmax = {}
-
+        E0,phi,df,T2 = 100.,0,0,.2
         first = False
         self.sigma = {}
         for pulse in self.DATADICT["PULSES"]:
                     IP[pulse][chan][ipm,:] = np.angle(ht)[clip::]
                     #############################################################
                     # Rotated amplitude
-                    [E0, df, phi, T2] = decay.quadratureDetect( ht.real, ht.imag, self.DATADICT[pulse]["TIMES"] )
+                    #if ipm != 0:
+                    [success, E0, df, phi, T2] = decay.quadratureDetect2( ht.real, ht.imag, self.DATADICT[pulse]["TIMES"], (E0,phi,df,T2))
+                    #[ E0, df, phi, T2] = decay.quadratureDetect( ht.real, ht.imag, self.DATADICT[pulse]["TIMES"] )
+                    #else:
+                    #    [success, E0, df, phi, T2] = decay.quadratureDetect2( ht.real, ht.imag, self.DATADICT[pulse]["TIMES"])
+                    #[success, E0, df, phi, T2] = decay.quadratureDetect( ht.real, ht.imag, self.DATADICT[pulse]["TIMES"] )
+                    print("success", success, "E0", E0, "phi", phi, "df", df, "T2", T2)
                     D = self.RotateAmplitude( ht.real, ht.imag, phi, df, self.DATADICT[pulse]["TIMES"] )
                     CA[pulse][chan][ipm,:] = D.imag[clip::]  # amplitude data 
                     NR[pulse][chan][ipm,:] = D.real[clip::]  # noise data
                 for ipm in range(0, self.DATADICT["nPulseMoments"]):
 
                     # Time since pulse rather than since record starts...
-                    if clip > 0:
-                        time_sp =  1e3 * (self.DATADICT[pulse]["TIMES"][clip-1::] - self.DATADICT[pulse]["PULSE_TIMES"][-1] )
-                    else:
-                        time_sp =  1e3 * (self.DATADICT[pulse]["TIMES"] - self.DATADICT[pulse]["PULSE_TIMES"][-1] )
+                    #if clip > 0:
+                    #    time_sp =  1e3 * (self.DATADICT[pulse]["TIMES"][clip:] - self.DATADICT[pulse]["PULSE_TIMES"][-1] )
+                    #else:
+                    time_sp =  1e3 * (self.DATADICT[pulse]["TIMES"] - self.DATADICT[pulse]["PULSE_TIMES"][-1] )
 
                     #GT, GD, GTT, sig_stack, isum      = rotate.gateIntegrate( self.DATADICT["CA"][pulse][chan][ipm,:], time_sp, gpd, self.sigma[pulse][chan], 1.5 )
                     #GT2, GP, GTT, sig_stack_err, isum = rotate.gateIntegrate( self.DATADICT["NR"][pulse][chan][ipm,:], time_sp, gpd, self.sigma[pulse][chan], 1.5 ) 
                     
-                    GT, GCA, GTT, sig_stack, isum  = rotate.gateIntegrate( self.DATADICT["CA"][pulse][chan][ipm,:], time_sp, gpd, self.sigma[pulse][chan], 1.5 )
-                    GT, GNR, GTT, sig_stack, isum  = rotate.gateIntegrate( self.DATADICT["NR"][pulse][chan][ipm,:], time_sp, gpd, self.sigma[pulse][chan], 1.5 )
-                    GT, GRE, GTT, sig_stack, isum  = rotate.gateIntegrate( self.DATADICT["RE"][pulse][chan][ipm,:], time_sp, gpd, self.sigma[pulse][chan], 1.5 )
-                    GT, GIM, GTT, sig_stack, isum  = rotate.gateIntegrate( self.DATADICT["IM"][pulse][chan][ipm,:], time_sp, gpd, self.sigma[pulse][chan], 1.5 )
-                    GT, GIP, GTT, sig_stack, isum  = rotate.gateIntegrate( self.DATADICT["IP"][pulse][chan][ipm,:], time_sp, gpd, self.sigma[pulse][chan], 1.5 )
+                    GT, GCA, GTT, sig_stack, isum  = rotate.gateIntegrate( self.DATADICT["CA"][pulse][chan][ipm], time_sp, gpd, self.sigma[pulse][chan], 1.5 )
+                    GT, GNR, GTT, sig_stack, isum  = rotate.gateIntegrate( self.DATADICT["NR"][pulse][chan][ipm], time_sp, gpd, self.sigma[pulse][chan], 1.5 )
+                    GT, GRE, GTT, sig_stack, isum  = rotate.gateIntegrate( self.DATADICT["RE"][pulse][chan][ipm], time_sp, gpd, self.sigma[pulse][chan], 1.5 )
+                    GT, GIM, GTT, sig_stack, isum  = rotate.gateIntegrate( self.DATADICT["IM"][pulse][chan][ipm], time_sp, gpd, self.sigma[pulse][chan], 1.5 )
+                    GT, GIP, GTT, sig_stack, isum  = rotate.gateIntegrate( self.DATADICT["IP"][pulse][chan][ipm], time_sp, gpd, self.sigma[pulse][chan], 1.5 )
                     
                     if ipm == 0:
                     #    self.GATED[chan]["DATA"]  = np.zeros( ( self.DATADICT["nPulseMoments"], len(GT)) )
                     #    self.GATED[chan]["ERR"]   = np.zeros( ( self.DATADICT["nPulseMoments"], len(GT)) )
                     #    self.GATED[chan]["SIGMA"] = np.zeros( ( self.DATADICT["nPulseMoments"], len(GT)) )
-                        self.GATED[chan]["CA"] = np.zeros( ( self.DATADICT["nPulseMoments"], len(GT)) )
-                        self.GATED[chan]["NR"] = np.zeros( ( self.DATADICT["nPulseMoments"], len(GT)) )
-                        self.GATED[chan]["RE"] = np.zeros( ( self.DATADICT["nPulseMoments"], len(GT)) )
-                        self.GATED[chan]["IM"] = np.zeros( ( self.DATADICT["nPulseMoments"], len(GT)) )
-                        self.GATED[chan]["IP"] = np.zeros( ( self.DATADICT["nPulseMoments"], len(GT)) )
+                        self.GATED[chan]["CA"] = np.zeros( ( self.DATADICT["nPulseMoments"], len(GT)-clip) )
+                        self.GATED[chan]["NR"] = np.zeros( ( self.DATADICT["nPulseMoments"], len(GT)-clip) )
+                        self.GATED[chan]["RE"] = np.zeros( ( self.DATADICT["nPulseMoments"], len(GT)-clip) )
+                        self.GATED[chan]["IM"] = np.zeros( ( self.DATADICT["nPulseMoments"], len(GT)-clip) )
+                        self.GATED[chan]["IP"] = np.zeros( ( self.DATADICT["nPulseMoments"], len(GT)-clip) )
                         self.GATED[chan]["isum"] = isum
 
                     #self.GATED[chan]["DATA"][ipm] = GD.real
-                    self.GATEDABSCISSA = GT
-                    self.GATEDWINDOW = GTT
+                    self.GATEDABSCISSA = GT[clip:]
+                    self.GATEDWINDOW = GTT[clip:]
                     #self.GATED[chan]["SIGMA"][ipm] =  sig_stack #_err # GP.real
                     #self.GATED[chan]["ERR"][ipm] =  GP.real
                     
-                    self.GATED[chan]["CA"][ipm] = GCA.real
-                    self.GATED[chan]["NR"][ipm] = GNR.real
-                    self.GATED[chan]["RE"][ipm] = GRE.real
-                    self.GATED[chan]["IM"][ipm] = GIM.real
-                    self.GATED[chan]["IP"][ipm] = GIP.real
+                    self.GATED[chan]["CA"][ipm] = GCA.real[clip:]
+                    self.GATED[chan]["NR"][ipm] = GNR.real[clip:]
+                    self.GATED[chan]["RE"][ipm] = GRE.real[clip:]
+                    self.GATED[chan]["IM"][ipm] = GIM.real[clip:]
+                    self.GATED[chan]["IP"][ipm] = GIP.real[clip:]
                     
                     percent = int(1e2* (float)((ipm)+ichan*self.DATADICT["nPulseMoments"]) / 
                                        (float)(self.DATADICT["nPulseMoments"] * len(self.DATADICT[pulse]["chan"])))
                     self.progressTrigger.emit(percent)
 
-                self.GATED[chan]["GTT"] = GTT
-                self.GATED[chan]["GT"] = GT
+                self.GATED[chan]["GTT"] = GTT[clip:]
+                self.GATED[chan]["GT"] = GT[clip:]
                 self.GATED[chan]["QQ"] = QQ
                 ichan += 1
         self.doneTrigger.emit() 
         self.doneTrigger.emit() 
         self.updateProcTrigger.emit()  
 
+    def loadGMRBinaryFID( self, rawfname, istack ):
+        """ Reads a single binary GMR file and fills into DATADICT
+        """
+
+        #################################################################################
+        # figure out key data indices
+        # Pulse        
+        nps  = (int)((self.prePulseDelay)*self.samp)
+        npul   = (int)(self.pulseLength[0]*self.samp) #+ 100 
+
+        # Data 
+        nds  = nps+npul+(int)((self.deadTime)*self.samp);        # indice pulse 1 data starts 
+        nd1 = (int)(1.*self.samp)                                # samples in first pulse
+
+        invGain = 1./self.RxGain        
+        invCGain = self.CurrentGain        
+
+        pulse = "Pulse 1"
+        chan = self.DATADICT[pulse]["chan"] 
+        rchan = self.DATADICT[pulse]["rchan"] 
+        
+        rawFile = open( rawfname, 'rb')
+
+        for ipm in range(self.nPulseMoments):
+            buf1 = rawFile.read(4)
+            buf2 = rawFile.read(4)
+                
+            N_chan = struct.unpack('>i', buf1 )[0]
+            N_samp = struct.unpack('>i', buf2 )[0]
+ 
+            T = N_samp * self.dt 
+            TIMES = np.arange(0, T, self.dt) - .0002 # small offset in GMR DAQ?
+
+            DATA = np.zeros([N_samp, N_chan+1])
+            for ichan in range(N_chan):
+                DATADUMP = rawFile.read(4*N_samp)
+                for irec in range(N_samp):
+                    DATA[irec,ichan] = struct.unpack('>f', DATADUMP[irec*4:irec*4+4])[0]
+                           
+            # Save into Data Cube 
+            for ichan in chan:
+                self.DATADICT["Pulse 1"][ichan][ipm][istack] = DATA[:,eval(ichan)+3][nds:nds+nd1] * invGain 
+                self.DATADICT["Pulse 1"]["TIMES"] = TIMES[nds:nds+nd1]
+                self.DATADICT["Pulse 1"]["CURRENT"][ipm][istack] = DATA[:,1][nps:nps+npul] * invCGain
+                self.DATADICT["Pulse 1"]["PULSE_TIMES"] = TIMES[nps:nps+npul] 
+
+            # reference channels?
+            for ichan in rchan:
+                self.DATADICT["Pulse 1"][ichan][ipm][istack] = DATA[:,eval(ichan)+3][nds:nds+nd1] * invGain 
+                self.DATADICT["Pulse 1"]["TIMES"] = TIMES[nds:nds+nd1]
+    
+    def loadGMRASCIIFID( self, rawfname, istack ):
+        """Based on the geoMRI instrument manufactured by VistaClara. Imports 
+        a suite of raw .lvm files with the following format (on one line)
+
+        time(s) DC_Bus/100(V) Current+/75(A)  Curr-/75(A)  Voltage+/200(V) \  
+        Ch1(V) Ch2(V) Ch3(V) Ch4(V)
+
+        Sampling rate is assumed at 50 kHz 
+        """
+        import pandas as pd 
+        #################################################################################
+        # figure out key data indices
+        # Pulse        
+        nps  = (int)((self.prePulseDelay)*self.samp)
+        npul   = (int)(self.pulseLength[0]*self.samp) #+ 100 
+
+        # Data 
+        nds  = nps+npul+(int)((self.deadTime)*self.samp);        # indice pulse 1 data starts 
+        nd1 = (int)(1.*self.samp) - nds                          # samples in first pulse
+        ndr = (int)(1.*self.samp)                                # samples in record 
+
+        invGain = 1./self.RxGain        
+        invCGain = self.CurrentGain        
+
+        pulse = "Pulse 1"
+        chan = self.DATADICT[pulse]["chan"] 
+        rchan = self.DATADICT[pulse]["rchan"] 
+            
+        T = 1.5 #N_samp * self.dt 
+        TIMES = np.arange(0, T, self.dt) - .0002 # small offset in GMR DAQ?
+        
+        self.DATADICT["Pulse 1"]["TIMES"] = TIMES[nds:nds+nd1]
+        self.DATADICT["Pulse 1"]["PULSE_TIMES"] = TIMES[nps:nps+npul]
+
+        # pandas is much faster than numpy for io
+        #DATA = np.loadtxt(rawfname)
+        DATA = pd.read_csv(rawfname, header=None, sep="\t").values
+        for ipm in range(self.nPulseMoments):
+            for ichan in np.append(chan,rchan):
+                self.DATADICT["Pulse 1"][ichan][ipm][istack] =  DATA[:, eval(ichan)+4][nds:(nds+nd1)] * invGain
+                self.DATADICT["Pulse 1"]["CURRENT"][ipm][istack] = DATA[:,2][nps:nps+npul] * invCGain
+            nds += ndr
+            nps += ndr 
+
+
     def loadFIDData(self, base, procStacks, chanin, rchanin, FIDProc, canvas, deadTime, plot):
         '''
-            Loads a GMR FID dataset, reads OLD ASCII files
+            Loads a GMR FID dataset, reads binary and ASCII format files 
         '''
-        import struct
+
         canvas.reAx3(True,False)
 
         chan = []
         self.deadTime       = deadTime       # instrument dead time before measurement
         self.samp = 50000.                   # in case this is a reproc, these might have 
         self.dt   = 1./self.samp             # changed
-        invGain = 1./self.RxGain        
-        invCGain = self.CurrentGain        
 
-        #################################################################################
-        # figure out key data indices
-        # Pulse        
-        nps  = (int)((self.prePulseDelay)*self.samp)
-        npul   = (int)(self.pulseLength[0]*self.samp) #+ 100 
-
-        # Data 
-        nds  = nps+npul+(int)((self.deadTime)*self.samp);        # indice pulse 1 data starts 
-        nd1 = (int)(1.*self.samp)                              # samples in first pulse
 
         #################################################################################
         # Data structures     
         ##############################################
         # Read in binary (.lvm) data
         iistack = 0
-        hack = False
+        fnames = []
         for istack in procStacks:
-            rawFile = open(base + "_" + str(istack) + ".lvm", 'rb')
-            if hack:
-                subFile = open(base + "sub" + "_" + str(istack) + ".lvm", 'wb')
-            for ipm in range(self.nPulseMoments):
-                
-                # frequency cycing modulation
-                #mod = (-1)**(ipm%2) * (-1)**(istack%2)
-                
-                buf1 = rawFile.read(4)
-                buf2 = rawFile.read(4)
-                
-                if hack:
-                    # hack to do some data analysis
-                    subFile.write(buf1)
-                    subFile.write(buf2)
-                    # end hack
-                
-                #N_chan = struct.unpack('>i', rawFile.read(4))[0]
-                #N_samp = struct.unpack('>i', rawFile.read(4))[0]
+            if self.nDAQVersion < 2.3:
+                #rawfname = base + "_" + str(istack) 
+                self.loadGMRASCIIFID( base + "_" + str(istack), istack )
+            else:
+                self.loadGMRBinaryFID( base + "_" + str(istack) + ".lvm", istack )
+                #fnames.append( base + "_" + str(istack) + ".lvm" )
                 
-                N_chan = struct.unpack('>i', buf1 )[0]
-                N_samp = struct.unpack('>i', buf2 )[0]
-        
- 
-                T = N_samp * self.dt 
-                TIMES = np.arange(0, T, self.dt) - .0002 # small offset in GMR DAQ?
-
-                DATA = np.zeros([N_samp, N_chan+1])
-                for ichan in range(N_chan):
-                    DATADUMP = rawFile.read(4*N_samp)
-                    #subFile.write(DATADUMP)
-                    for irec in range(N_samp):
-                        DATA[irec,ichan] = struct.unpack('>f', DATADUMP[irec*4:irec*4+4])[0]
+            percent = (int) (1e2*((float)((iistack*self.nPulseMoments+ipm+1))  / (len(procStacks)*self.nPulseMoments)))
+            self.progressTrigger.emit(percent) 
+            iistack += 1
 
-                if hack:                       
-                    # hack to do some data analysis (array survey)
-                    for ichan in range(5):
-                        for irec in range(N_samp): 
-                            bdata = struct.pack( '>f', DATA[irec,ichan] )               
-                            subFile.write(bdata)
-                    for ichan in range(5,6):
-                        for irec in range(N_samp): 
-                            bdata = struct.pack( '>f', DATA[irec,ichan] + DATA[irec,ichan+2] )               
-                            #bdata = struct.pack( '>f', .23 )               
-                            subFile.write(bdata)
-                    for ichan in range(6,N_chan):
-                        for irec in range(N_samp): 
-                            bdata = struct.pack( '>f', DATA[irec,ichan] )               
-                            subFile.write(bdata)
+        # multiprocessing load data
+        #info = {}
+        #info["prePulseDelay"] = self.prePulseDelay
+        #with multiprocessing.Pool() as pool: 
+        #    results = pool.starmap( loadGMRBinaryFID, zip(itertools.repeat(self), fnames, info ) ) # zip(np.tile(vc, (ns, 1)), np.tile(vgc, (ns,1)), itertools.repeat(sys.argv[1]), itertools.repeat(sys.argv[2]), EPS_CMR))
 
-                if plot: 
+        # Plotting
+        if plot: 
+            iistack = 0
+            for istack in procStacks:
+                for ipm in range(self.nPulseMoments):
                     canvas.ax1.clear()
                     canvas.ax2.clear()
                     canvas.ax3.clear()
                            
-                # Save into Data Cube 
-                for ichan in chan:
-                    self.DATADICT["Pulse 1"][ichan][ipm][istack] = DATA[:,eval(ichan)+3][nds:nds+nd1] * invGain 
-                    self.DATADICT["Pulse 1"]["TIMES"] = TIMES[nds:nds+nd1]
-                    self.DATADICT["Pulse 1"]["CURRENT"][ipm][istack] = DATA[:,1][nps:nps+npul] * invCGain
-                    self.DATADICT["Pulse 1"]["PULSE_TIMES"] = TIMES[nps:nps+npul] 
-                    if plot:
-                        #canvas.ax2.plot(self.DATADICT["Pulse 1"]["TIMES"],       self.DATADICT["Pulse 1"][ichan][ipm][istack], label="Pulse 1 FID data ch. "+str(ichan)) #, color='blue')
+                    for ichan in chan:
                         canvas.ax1.plot(self.DATADICT["Pulse 1"]["PULSE_TIMES"], self.DATADICT["Pulse 1"]["CURRENT"][ipm][istack] , color='black')
                         canvas.ax3.plot(self.DATADICT["Pulse 1"]["TIMES"],       self.DATADICT["Pulse 1"][ichan][ipm][istack], label="Pulse 1 FID data ch. "+str(ichan)) #, color='blue')
-                        #canvas.draw()
 
-                # plot reference channels?
-                for ichan in rchan:
-                    self.DATADICT["Pulse 1"][ichan][ipm][istack] = DATA[:,eval(ichan)+3][nds:nds+nd1] * invGain 
-                    self.DATADICT["Pulse 1"]["TIMES"] = TIMES[nds:nds+nd1]
-                    if plot:
+                    for ichan in rchan:
                         canvas.ax2.plot(self.DATADICT["Pulse 1"]["TIMES"], self.DATADICT["Pulse 1"][ichan][ipm][istack], label="Pulse 1 FID ref ch. "+str(ichan)) #, color='blue')
 
-                if plot:
                     canvas.ax3.legend(prop={'size':6})
                     canvas.ax2.legend(prop={'size':6})
                     
                     canvas.ax2.set_xlabel("time [s]", fontsize=8)
                     canvas.ax2.ticklabel_format(style='sci', scilimits=(0,0), axis='y') 
                     canvas.ax3.ticklabel_format(style='sci', scilimits=(0,0), axis='y') 
-
                     canvas.draw()
+
                 percent = (int) (1e2*((float)((iistack*self.nPulseMoments+ipm+1))  / (len(procStacks)*self.nPulseMoments)))
-                #self.progressTrigger.emit(percent) 
                 self.progressTrigger.emit(percent) 
-
-            iistack += 1
-
+                iistack += 1
 
         self.enableDSP()    
         self.doneTrigger.emit()
-
     
     def load4PhaseT1Data(self, base, procStacks, chan, rchan, FIDProc, canvas, deadTime, plot): 
 

akvo/tressel/rotate.py

         # decimate
     # blind decimation
     # 1 instead of T 
-    irsamp = (T) * int(  (1./vL) / dt) # real 
+    irsamp = int(T) * int(  (1./vL) / dt) # real 
     iisamp =       int(  ((1./vL)/ dt) * ( .5*np.pi / (2.*np.pi) ) ) # imaginary
    
 
     #############################################################
     ## In-phase 
     #2*np.cos(wL*t)  
-    dw = -2.*np.pi*2
+    dw = 0 # -2.*np.pi*2
     Q = signal.filtfilt(b, a, xn*2*np.cos((wL+dw)*t))  # X
     I = signal.filtfilt(b, a, xn*2*np.sin((wL+dw)*t))  # Y
 
 
 def gateIntegrate(T2D, T2T, gpd, sigma, stackEfficiency=2.):
     """ Gate integrate the signal to gpd, gates per decade
+        T2D = the time series to gate integrate, complex 
+        T2T = the abscissa values 
+        gpd = gates per decade 
+        sigma = estimate of standard deviation for theoretical gate noise 
+        stackEfficiency = exponential in theoretical gate noise, 2 represents ideal stacking
     """
     
     # use artificial time gates so that early times are fully captured
     T2TD = T2T[0] - (T2T[1]-T2T[0])
     T2T -= T2TD
     
-    # calculate total number of decades
-    nd = np.log10(T2T[-1]/T2T[0]) #np.log10(self.T2T[-1]) -  np.log10(self.T2T[-1])
+    #####################################
+    # calculate total number of decades #
+    # windows edges are approximate until binning but will be adjusted to reflect data timing, this 
+    # primarily impacts bins with a few samples  
+    nd = np.log10(T2T[-1]/T2T[0])               
     tdd = np.logspace( np.log10(T2T[0]), np.log10(T2T[-1]), (int)(gpd*nd)+1, base=10, endpoint=True) 
-    tdl = tdd[0:-1]     # these are left edges
-    tdr = tdd[1::]      # these are left edges
-    td = (tdl+tdr) / 2. # window centres
+    tdl = tdd[0:-1]                 # approximate window left edges
+    tdr = tdd[1::]                  # approximate window right edges
+    td = (tdl+tdr) / 2.             # approximate window centres
 
-    Vars = np.ones( len(td) ) * sigma**2 #* .15**2
+
+    Vars = np.zeros( len(td) ) 
     htd = np.zeros( len(td), dtype=complex )
-    isum = np.zeros( len(td) )
-    ii = 0
+    isum = np.zeros( len(td), dtype=int )  
 
-    SIGSTACK = {}
-    SIGSTACK[ii]= []
+    ii = 0
     for itd in range(len(T2T)):
         if ( T2T[itd] > tdr[ii] ):
-            if (ii < len(td)-1):
-                ii += 1
-                SIGSTACK[ii] = []
-            else:
-                #print "overshoot??", ii
-                break
+            ii += 1
+            # correct window edges to centre about data 
+            tdr[ii-1] = (T2T[itd-1]+T2T[itd])*.5 
+            tdl[ii  ] = (T2T[itd-1]+T2T[itd])*.5
         isum[ii] += 1
         htd[ii] += T2D[ itd ]
-        SIGSTACK[ii].append(T2D[itd])
         Vars[ii] += sigma**2
+        
+    td = (tdl+tdr) / 2.             # actual window centres
+    sigma2 = np.sqrt( Vars * ((1/(isum))**stackEfficiency) ) 
 
-    sigma = np.sqrt( Vars  * (1/isum)**stackEfficiency ) 
-    for ii in range(len(td)):
-        if len(SIGSTACK[ii]) > 30:
-            sigma[ii] = np.var(SIGSTACK[ii]) / ((len(SIGSTACK[ii])-1)**(1/stackEfficiency))
-            #sigma[ii] = np.std(SIGSTACK[ii]) * ( 1./(len(SIGSTACK[ii]))**stackEfficiency)
+    # Reset abscissa where isum == 1 
+    # when there is no windowing going on 
+    td[isum==1] = T2T[0:len(td)][isum==1]
 
-    # RESET times where isum == 1
-    ii = 0
-    while (isum[ii] == 1):
-        td[ii] = T2T[ii]
-        ii += 1
+    tdd = np.append(tdl, tdr[-1])
 
-    htd /= isum
-    T2T += T2TD  
-    return td+T2TD, htd, tdd+T2TD, sigma, isum  # centre abscissa, data, window edges, error  
+    htd /= isum # average
+    T2T += T2TD 
+    return td+T2TD, htd, tdd+T2TD, sigma2, isum  # centre abscissa, data, window edges, error 
 
 if __name__ == "__main__":
 

setup.py

           'numpy',
           'PyQt5',
           'pyyaml',
+          'pandas',
           'pyqt-distutils',
           'cmocean'
       ],