Browse Source

MRS MIDI II support added

master
Trevor Irons 2 years ago
parent
commit
33aacb67ca

+ 201
- 13
akvo/gui/akvoGUI.py View File

@@ -120,6 +120,7 @@ class ApplicationWindow(QtWidgets.QMainWindow):
120 120
         # Menu items #
121 121
         ##############
122 122
         self.ui.actionOpen_GMR.triggered.connect(self.openGMRRAWDataset)
123
+        self.ui.actionLoad_MIDI.triggered.connect(self.loadMIDI2Dataset)
123 124
         self.ui.actionSave_Preprocessed_Dataset.triggered.connect(self.SavePreprocess)
124 125
         self.ui.actionExport_Preprocessed_Dataset.triggered.connect(self.ExportPreprocess)
125 126
         self.ui.actionExport_Preprocessed_Dataset.setEnabled(False)
@@ -129,7 +130,7 @@ class ApplicationWindow(QtWidgets.QMainWindow):
129 130
         ###########
130 131
         # Buttons #
131 132
         ###########
132
-        self.ui.loadDataPushButton.pressed.connect(self.loadRAW) 
133
+        #self.ui.loadDataPushButton.pressed.connect(self.loadRAW) 
133 134
         self.ui.sumDataGO.pressed.connect( self.sumDataChans )
134 135
         self.ui.bandPassGO.pressed.connect( self.bandPassFilter )
135 136
         self.ui.filterDesignPushButton.pressed.connect( self.designFilter )
@@ -884,7 +885,79 @@ class ApplicationWindow(QtWidgets.QMainWindow):
884 885
         self.RAWDataProc.enableDSPTrigger.connect(self.enableDSP)
885 886
         self.RAWDataProc.doneTrigger.connect(self.doneStatus)
886 887
         self.RAWDataProc.updateProcTrigger.connect(self.updateProc)
888
+    
889
+    def loadMIDI2Dataset (self):
890
+        """ Opens a MIDI file and extracts header info
891
+        """
892
+        try:
893
+            with open('.midi2.last.path') as f: 
894
+                fpath = f.readline()  
895
+                pass
896
+        except IOError as e:
897
+            fpath = '.'
898
+         
899
+        self.headerstr = QtWidgets.QFileDialog.getExistingDirectory(self, 'Load MIDI 2 Directory', fpath)
900
+        self.ui.headerFileTextBrowser.clear()
901
+        self.ui.headerFileTextBrowser.append(self.headerstr)
902
+        
903
+        if len(self.headerstr) == 0:
904
+            return 
905
+
906
+        # TODO, should rename this class, use MIDI path  
907
+        self.connectGMRDataProcessor()
908
+        self.RAWDataProc.readMIDI2Header(str(self.headerstr))
887 909
 
910
+        # look in the directory for all the files 
911
+        self.ui.loadDataPushButton.pressed.connect(self.loadMIDI2) 
912
+
913
+        # If we got this far, enable all the widgets
914
+        self.ui.lcdNumberTauPulse1.setEnabled(True)
915
+        self.ui.lcdNumberNuTx.setEnabled(True)
916
+        self.ui.lcdNumberTuneuF.setEnabled(True)
917
+        self.ui.lcdNumberSampFreq.setEnabled(True)
918
+        self.ui.lcdNumberNQ.setEnabled(True)
919
+
920
+        self.ui.headerFileBox.setEnabled(True)
921
+        self.ui.headerFileBox.setChecked( True )
922
+        self.ui.headerBox2.setVisible(True) 
923
+        self.ui.inputRAWParametersBox.setEnabled(True)
924
+        self.ui.loadDataPushButton.setEnabled(True)
925
+         
926
+        # make plots as you import the dataset
927
+        self.ui.plotImportCheckBox.setEnabled(True)
928
+        self.ui.plotImportCheckBox.setChecked(True)
929
+         
930
+        # Update info from the header into the GUI
931
+        self.ui.pulseTypeTextBrowser.clear()
932
+        self.ui.pulseTypeTextBrowser.append(self.RAWDataProc.pulseType)
933
+        self.ui.lcdNumberNuTx.display(self.RAWDataProc.transFreq)
934
+        self.ui.lcdNumberTauPulse1.display(1e3*self.RAWDataProc.pulseLength[0])
935
+        self.ui.lcdNumberTuneuF.display(self.RAWDataProc.TuneCapacitance)
936
+        self.ui.lcdNumberSampFreq.display(self.RAWDataProc.samp)
937
+        self.ui.lcdNumberNQ.display(self.RAWDataProc.nPulseMoments)
938
+        self.ui.DeadTimeSpinBox.setValue(1e3*self.RAWDataProc.deadTime)
939
+        self.ui.CentralVSpinBox.setValue( self.RAWDataProc.transFreq )
940
+
941
+        # set the B0 field according to Tx as an initial guess
942
+        self.ui.intensitySpinBox.setValue( self.RAWDataProc.transFreq/GAMMAH )           
943
+ 
944
+        if self.RAWDataProc.pulseType != "FID":
945
+            self.ui.lcdNumberTauPulse2.setEnabled(1)
946
+            self.ui.lcdNumberTauPulse2.display(1e3*self.RAWDataProc.pulseLength[1])
947
+            self.ui.lcdNumberTauDelay.setEnabled(1)
948
+            self.ui.lcdNumberTauDelay.display(1e3*self.RAWDataProc.interpulseDelay)
949
+        
950
+        self.ui.FIDProcComboBox.clear() 
951
+        if self.RAWDataProc.pulseType == "4PhaseT1" or self.RAWDataProc.pulseType == "T1":
952
+            self.ui.FIDProcComboBox.insertItem(0, "Pulse 1") 
953
+            self.ui.FIDProcComboBox.insertItem(1, "Pulse 2") 
954
+            self.ui.FIDProcComboBox.insertItem(2, "Both")    
955
+            self.ui.FIDProcComboBox.setCurrentIndex (1)
956
+        elif self.RAWDataProc.pulseType == "FID":
957
+            self.ui.FIDProcComboBox.insertItem(0, "Pulse 1") 
958
+            self.ui.FIDProcComboBox.setCurrentIndex (0)
959
+    
960
+ 
888 961
     def openGMRRAWDataset(self):
889 962
         """ Opens a GMR header file
890 963
         """
@@ -913,6 +986,9 @@ class ApplicationWindow(QtWidgets.QMainWindow):
913 986
 
914 987
         self.connectGMRDataProcessor()
915 988
         self.RAWDataProc.readHeaderFile(str(self.headerstr))
989
+        
990
+        # make sure we will use GMR path 
991
+        self.ui.loadDataPushButton.pressed.connect(self.loadRAW) 
916 992
 
917 993
         # If we got this far, enable all the widgets
918 994
         self.ui.lcdNumberTauPulse1.setEnabled(True)
@@ -1082,6 +1158,12 @@ class ApplicationWindow(QtWidgets.QMainWindow):
1082 1158
         INFO["nDAQVersion"] = self.RAWDataProc.nDAQVersion
1083 1159
         INFO["log"] = yaml.dump( self.YamlNode )  
1084 1160
 
1161
+        # 1.6.4 and on 
1162
+        INFO["Instrument"] = self.RAWDataProc.Instrument 
1163
+        if self.RAWDataProc.Instrument == "MIDI 2":
1164
+            INFO["MIDIGain"] = self.RAWDataProc.MIDIGain
1165
+            INFO["datadir"] = self.RAWDataProc.datadir
1166
+
1085 1167
         TXRX = []
1086 1168
         for ir in range(0, self.ui.txRxTable.rowCount() ):
1087 1169
             txrx = []
@@ -1096,12 +1178,14 @@ class ApplicationWindow(QtWidgets.QMainWindow):
1096 1178
         if "Gate integrate" in self.YamlNode.Stacking.keys():
1097 1179
             INFO["GATED"] = self.RAWDataProc.GATED
1098 1180
 
1099
-        print("META SAVE")
1100
-        print("INFO log", INFO["log"])
1181
+        #print("META SAVE")
1182
+        #print("INFO log", INFO["log"])
1101 1183
 
1102 1184
         self.RAWDataProc.DATADICT["INFO"] = INFO 
1103 1185
 
1104 1186
         pickle.dump(self.RAWDataProc.DATADICT, save)
1187
+        #pickle.dump(self.RAWDataProc, save) # doesn't work :-( 
1188
+
1105 1189
         save.close()
1106 1190
 
1107 1191
     # Export XML file suitable for USGS ScienceBase Data Release
@@ -1296,7 +1380,109 @@ class ApplicationWindow(QtWidgets.QMainWindow):
1296 1380
         self.RAWDataProc.doneTrigger.connect(self.doneStatus)
1297 1381
         
1298 1382
         self.enableAll()
1383
+
1384
+    def loadMIDI2(self):
1385
+
1386
+        #################################################
1387
+        # Check to make sure we are ready to process
1388
+
1389
+        # Header
1390
+        if self.RAWDataProc == None:
1391
+            err_msg = "You need to load a header first."
1392
+            reply = QtWidgets.QMessageBox.critical(self, 'Error', 
1393
+                err_msg) #, QtWidgets.QMessageBox.Yes, QtWidgets.QMessageBox.No)
1394
+            return
1395
+        
1396
+        # Stacks 
1397
+        try:
1398
+            self.procStacks = np.array(eval(str("np.r_["+self.ui.stacksLineEdit.text())+"]"))
1399
+        except:
1400
+            err_msg = "You need to set your stacks correctly.\n" + \
1401
+                      "This should be a Python Numpy interpretable list\n" + \
1402
+                      "of stack indices. For example 1:24 or 1:4,8:24"
1403
+            QtWidgets.QMessageBox.critical(self, 'Error', err_msg) 
1404
+            return
1405
+
1406
+        # Data Channels
1407
+        #Chan = np.arange(0,9,1)
1408
+        try:
1409
+            self.dataChan = np.array(eval(str("np.r_["+self.ui.dataChanLineEdit.text())+"]"))
1410
+        except:
1411
+            #QMessageBox messageBox;
1412
+            #messageBox.critical(0,"Error","An error has occured !");
1413
+            #messageBox.setFixedSize(500,200);
1414
+            #quit_msg = "Are you sure you want to exit the program?"
1415
+            #reply = QtWidgets.QMessageBox.question(self, 'Message', 
1416
+            #    quit_msg, QtWidgets.QMessageBox.Yes, QtWidgets.QMessageBox.No)
1417
+            err_msg = "You need to set your data channels correctly.\n" + \
1418
+                      "This should be a Python Numpy interpretable list\n" + \
1419
+                      "of indices. For example 1 or 1:3 or 1:3 5\n\n" + \
1420
+                      "valid GMR data channels fall between 1 and 8. Note that\n" +\
1421
+                      "1:3 is not inclusive of 3 and is the same as 1,2 "
1422
+            reply = QtWidgets.QMessageBox.critical(self, 'Error', 
1423
+                err_msg) #, QtWidgets.QMessageBox.Yes, QtWidgets.QMessageBox.No)
1424
+            return
1425
+        #############################
1426
+        # Reference Channels
1427
+        # TODO make sure no overlap between data and ref channels
1428
+        self.refChan = np.array( () )
1429
+        if str(self.ui.refChanLineEdit.text()): # != "none":
1430
+            try:
1431
+                self.refChan = np.array(eval(str("np.r_["+self.ui.refChanLineEdit.text())+"]"))
1432
+            except:
1433
+                err_msg = "You need to set your reference channels correctly.\n" + \
1434
+                      "This should be a Python Numpy interpretable list\n" + \
1435
+                      "of indices. For example 1 or 1:3 or 1:3 5\n\n" + \
1436
+                      "valid GMR data channels fall between 1 and 8. Note that\n" +\
1437
+                      "1:3 is not inclusive of 3 and is the same as 1,2 "
1438
+                QtWidgets.QMessageBox.critical(self, 'Error', err_msg) 
1439
+                return
1440
+
1441
+        #####################################################
1442
+        # Load data
1443
+        
1444
+        self.lock("loading MIDI 2 dataset")
1445
+        self.procThread = thread.start_new_thread(self.RAWDataProc.loadMIDI2, \
1446
+                (str(self.headerstr), self.procStacks, self.dataChan, self.refChan, \
1447
+                 str(self.ui.FIDProcComboBox.currentText()), self.ui.mplwidget, \
1448
+                1e-3 * self.ui.DeadTimeSpinBox.value( ), self.ui.plotImportCheckBox.isChecked() )) #, self)) 
1449
+
1450
+        self.YamlNode.Import["MIDI Header"] = self.headerstr
1451
+        self.YamlNode.Import["opened"] = datetime.datetime.now().isoformat() 
1452
+        self.YamlNode.Import["pulse Type"] = str(self.RAWDataProc.pulseType) 
1453
+        self.YamlNode.Import["stacks"] = self.procStacks.tolist() 
1454
+        self.YamlNode.Import["data channels"] = self.dataChan.tolist()  
1455
+        self.YamlNode.Import["reference channels"] = self.refChan.tolist() 
1456
+        self.YamlNode.Import["pulse records"] = str(self.ui.FIDProcComboBox.currentText())  
1457
+        self.YamlNode.Import["instrument dead time"] = (1e-3 * self.ui.DeadTimeSpinBox.value( ))    
1458
+
1459
+        self.Log (  )     
1460
+        
1461
+        # enable META tab 
1462
+        self.ui.METATab.setEnabled(1)
1463
+        self.ui.siteBox.setEnabled(1)
1464
+
1465
+        # should be already done
1466
+#        QtCore.QObject.connect(self.RAWDataProc, QtCore.SIGNAL("updateProgress(int)"), self.updateProgressBar)
1467
+#        QtCore.QObject.connect(self.RAWDataProc, QtCore.SIGNAL("enableDSP()"), self.enableDSP)
1468
+#        QtCore.QObject.connect(self.RAWDataProc, QtCore.SIGNAL("doneStatus()"), self.doneStatus)
1469
+
1470
+        #self.ui.ProcessedBox.setEnabled(True)
1471
+        self.ui.lcdNumberFID1Length.setEnabled(1)
1472
+        self.ui.lcdNumberFID2Length.setEnabled(1)
1473
+        self.ui.lcdNumberResampFreq.setEnabled(1)
1474
+        self.ui.lcdTotalDeadTime.setEnabled(1)
1475
+
1476
+        self.ui.lcdTotalDeadTime.display( self.ui.DeadTimeSpinBox.value( ) )
1477
+        #self.ui.lcdTotalDeadTime.display( round(1e3*(self.RAWDataProc.DATADICT["Pulse 1"]["TIMES"][0]-self.RAWDataProc.DATADICT["Pulse 1"]["PULSE_TIMES"][-1]), 3) )
1478
+        
1479
+        #self.ui.lcdNumberFID1Length.display(0)
1480
+        #self.ui.lcdNumberFID2Length.display(0)
1481
+        #self.ui.lcdNumberResampFreq.display( self.RAWDataProc.samp )
1299 1482
  
1483
+        self.mpl_toolbar = NavigationToolbar2QT(self.ui.mplwidget, self.ui.mplwidget)
1484
+        self.ui.mplwidget.draw()
1485
+
1300 1486
     def loadRAW(self):
1301 1487
 
1302 1488
         #################################################
@@ -1305,8 +1491,8 @@ class ApplicationWindow(QtWidgets.QMainWindow):
1305 1491
         # Header
1306 1492
         if self.RAWDataProc == None:
1307 1493
             err_msg = "You need to load a header first."
1308
-            reply = QtGui.QMessageBox.critical(self, 'Error', 
1309
-                err_msg) #, QtGui.QMessageBox.Yes, QtGui.QMessageBox.No)
1494
+            reply = QtWidgets.QMessageBox.critical(self, 'Error', 
1495
+                err_msg) #, QtWidgets.QMessageBox.Yes, QtWidgets.QMessageBox.No)
1310 1496
             return
1311 1497
         
1312 1498
         # Stacks 
@@ -1316,7 +1502,7 @@ class ApplicationWindow(QtWidgets.QMainWindow):
1316 1502
             err_msg = "You need to set your stacks correctly.\n" + \
1317 1503
                       "This should be a Python Numpy interpretable list\n" + \
1318 1504
                       "of stack indices. For example 1:24 or 1:4,8:24"
1319
-            QtGui.QMessageBox.critical(self, 'Error', err_msg) 
1505
+            QtWidgets.QMessageBox.critical(self, 'Error', err_msg) 
1320 1506
             return
1321 1507
 
1322 1508
         # Data Channels
@@ -1328,15 +1514,15 @@ class ApplicationWindow(QtWidgets.QMainWindow):
1328 1514
             #messageBox.critical(0,"Error","An error has occured !");
1329 1515
             #messageBox.setFixedSize(500,200);
1330 1516
             #quit_msg = "Are you sure you want to exit the program?"
1331
-            #reply = QtGui.QMessageBox.question(self, 'Message', 
1332
-            #    quit_msg, QtGui.QMessageBox.Yes, QtGui.QMessageBox.No)
1517
+            #reply = QtWidgets.QMessageBox.question(self, 'Message', 
1518
+            #    quit_msg, QtWidgets.QMessageBox.Yes, QtWidgets.QMessageBox.No)
1333 1519
             err_msg = "You need to set your data channels correctly.\n" + \
1334 1520
                       "This should be a Python Numpy interpretable list\n" + \
1335 1521
                       "of indices. For example 1 or 1:3 or 1:3 5\n\n" + \
1336 1522
                       "valid GMR data channels fall between 1 and 8. Note that\n" +\
1337 1523
                       "1:3 is not inclusive of 3 and is the same as 1,2 "
1338
-            reply = QtGui.QMessageBox.critical(self, 'Error', 
1339
-                err_msg) #, QtGui.QMessageBox.Yes, QtGui.QMessageBox.No)
1524
+            reply = QtWidgets.QMessageBox.critical(self, 'Error', 
1525
+                err_msg) #, QtWidgets.QMessageBox.Yes, QtWidgets.QMessageBox.No)
1340 1526
             return
1341 1527
         #############################
1342 1528
         # Reference Channels
@@ -1351,13 +1537,13 @@ class ApplicationWindow(QtWidgets.QMainWindow):
1351 1537
                       "of indices. For example 1 or 1:3 or 1:3 5\n\n" + \
1352 1538
                       "valid GMR data channels fall between 1 and 8. Note that\n" +\
1353 1539
                       "1:3 is not inclusive of 3 and is the same as 1,2 "
1354
-                QtGui.QMessageBox.critical(self, 'Error', err_msg) 
1540
+                QtWidgets.QMessageBox.critical(self, 'Error', err_msg) 
1355 1541
                 return
1356 1542
 
1357 1543
         #####################################################
1358 1544
         # Load data
1359
-
1360 1545
         self.lock("loading RAW GMR dataset")
1546
+        
1361 1547
 
1362 1548
         if self.RAWDataProc.pulseType == "FID":
1363 1549
             self.procThread = thread.start_new_thread(self.RAWDataProc.loadFIDData, \
@@ -1466,6 +1652,7 @@ class ApplicationWindow(QtWidgets.QMainWindow):
1466 1652
         # Adaptive filtering 
1467 1653
         self.ui.adaptBox.setEnabled(True)
1468 1654
         self.ui.adaptBox.setChecked(True)
1655
+        self.ui.plotRLS.setEnabled(True)
1469 1656
         
1470 1657
         # FD Adaptive filtering 
1471 1658
         self.ui.adaptFDBox.setEnabled(True)
@@ -1648,6 +1835,7 @@ class ApplicationWindow(QtWidgets.QMainWindow):
1648 1835
                 self.ui.adaptTruncateSpinBox.value(), \
1649 1836
                 self.ui.adaptMuSpinBox.value(), \
1650 1837
                 str(self.ui.PCAComboBox.currentText()), \
1838
+                self.ui.plotRLS.isChecked(), \
1651 1839
                 self.ui.mplwidget))
1652 1840
 
1653 1841
     def sumDataChans(self): 
@@ -1661,7 +1849,7 @@ class ApplicationWindow(QtWidgets.QMainWindow):
1661 1849
 
1662 1850
         self.dataChan = [self.dataChan[0]]
1663 1851
         self.ui.sumDataBox.setEnabled(False)    
1664
-        thread.start_new_thread( self.RAWDataProc.sumData, ( self.ui.mplwidget, 7 ) )
1852
+        thread.start_new_thread( self.RAWDataProc.sumData, ( self.ui.mplwidget, self.ui.sumType.currentText(), self.ui.sumAll.isChecked() ) )
1665 1853
  
1666 1854
     def adaptFilterFD(self):
1667 1855
         self.lock("FD noise cancellation filter")

+ 122
- 77
akvo/gui/main.ui View File

@@ -1576,7 +1576,7 @@ background: dark grey;
1576 1576
                <bool>false</bool>
1577 1577
               </property>
1578 1578
               <property name="sizePolicy">
1579
-               <sizepolicy hsizetype="Fixed" vsizetype="Expanding">
1579
+               <sizepolicy hsizetype="Preferred" vsizetype="Expanding">
1580 1580
                 <horstretch>0</horstretch>
1581 1581
                 <verstretch>0</verstretch>
1582 1582
                </sizepolicy>
@@ -1608,7 +1608,7 @@ background: dark grey;
1608 1608
                 </widget>
1609 1609
                </item>
1610 1610
                <item row="0" column="1">
1611
-                <widget class="QComboBox" name="comboBox">
1611
+                <widget class="QComboBox" name="sumType">
1612 1612
                  <item>
1613 1613
                   <property name="text">
1614 1614
                    <string>sum</string>
@@ -1646,6 +1646,22 @@ background: dark grey;
1646 1646
                  </property>
1647 1647
                 </widget>
1648 1648
                </item>
1649
+               <item row="1" column="0">
1650
+                <widget class="QCheckBox" name="sumAll">
1651
+                 <property name="sizePolicy">
1652
+                  <sizepolicy hsizetype="Preferred" vsizetype="Fixed">
1653
+                   <horstretch>0</horstretch>
1654
+                   <verstretch>0</verstretch>
1655
+                  </sizepolicy>
1656
+                 </property>
1657
+                 <property name="toolTip">
1658
+                  <string>&lt;html&gt;&lt;head/&gt;&lt;body&gt;&lt;p&gt;In case of more than two data channels, would you like to sum all of the channels together?&lt;/p&gt;&lt;/body&gt;&lt;/html&gt;</string>
1659
+                 </property>
1660
+                 <property name="text">
1661
+                  <string>Sum All</string>
1662
+                 </property>
1663
+                </widget>
1664
+               </item>
1649 1665
               </layout>
1650 1666
              </widget>
1651 1667
             </item>
@@ -1947,13 +1963,6 @@ background: dark grey;
1947 1963
                  </property>
1948 1964
                 </widget>
1949 1965
                </item>
1950
-               <item row="15" column="0">
1951
-                <widget class="Line" name="line_7">
1952
-                 <property name="orientation">
1953
-                  <enum>Qt::Horizontal</enum>
1954
-                 </property>
1955
-                </widget>
1956
-               </item>
1957 1966
                <item row="11" column="0">
1958 1967
                 <widget class="QLabel" name="label_27">
1959 1968
                  <property name="text">
@@ -2156,6 +2165,13 @@ background: dark grey;
2156 2165
                  </property>
2157 2166
                 </widget>
2158 2167
                </item>
2168
+               <item row="15" column="0">
2169
+                <widget class="Line" name="line_7">
2170
+                 <property name="orientation">
2171
+                  <enum>Qt::Horizontal</enum>
2172
+                 </property>
2173
+                </widget>
2174
+               </item>
2159 2175
               </layout>
2160 2176
              </widget>
2161 2177
             </item>
@@ -2195,6 +2211,69 @@ background: dark grey;
2195 2211
                <bool>true</bool>
2196 2212
               </property>
2197 2213
               <layout class="QGridLayout" name="gridLayout_12">
2214
+               <item row="2" column="1">
2215
+                <widget class="QDoubleSpinBox" name="adaptTruncateSpinBox">
2216
+                 <property name="toolTip">
2217
+                  <string>&lt;html&gt;&lt;head/&gt;&lt;body&gt;&lt;p&gt;This filter is a time-domain filter that takes some time to get going. Time-domain filters do a better job compared to frequency-domain filters in the presence of non-stationary noise. &lt;/p&gt;&lt;p&gt;The filter is run backwards, so often the late times will not be cancelled as well. You may trim records off the back using this input. &lt;/p&gt;&lt;/body&gt;&lt;/html&gt;</string>
2218
+                 </property>
2219
+                 <property name="whatsThis">
2220
+                  <string>&lt;html&gt;&lt;head/&gt;&lt;body&gt;&lt;p&gt;This filter is a time-domain filter that takes some time to get going. Time-domain filters do a better job compared to frequency-domain filters in the presence of non-stationary noise.  &lt;/p&gt;&lt;p&gt;The filter is run backwards, so often the late times will not be cancelled as well. You may trim records off the back using this input. &lt;/p&gt;&lt;/body&gt;&lt;/html&gt;</string>
2221
+                 </property>
2222
+                 <property name="maximum">
2223
+                  <double>1000.000000000000000</double>
2224
+                 </property>
2225
+                 <property name="value">
2226
+                  <double>800.000000000000000</double>
2227
+                 </property>
2228
+                </widget>
2229
+               </item>
2230
+               <item row="1" column="1">
2231
+                <widget class="QDoubleSpinBox" name="adaptLambdaSpinBox">
2232
+                 <property name="toolTip">
2233
+                  <string>Forgetting factor, how quickly does the filter adapt.</string>
2234
+                 </property>
2235
+                 <property name="decimals">
2236
+                  <number>3</number>
2237
+                 </property>
2238
+                 <property name="minimum">
2239
+                  <double>0.950000000000000</double>
2240
+                 </property>
2241
+                 <property name="maximum">
2242
+                  <double>1.000000000000000</double>
2243
+                 </property>
2244
+                 <property name="value">
2245
+                  <double>0.990000000000000</double>
2246
+                 </property>
2247
+                </widget>
2248
+               </item>
2249
+               <item row="2" column="0">
2250
+                <widget class="QLabel" name="label_46">
2251
+                 <property name="text">
2252
+                  <string>Truncate [ms]</string>
2253
+                 </property>
2254
+                </widget>
2255
+               </item>
2256
+               <item row="0" column="2">
2257
+                <widget class="QLabel" name="label_51">
2258
+                 <property name="text">
2259
+                  <string>Mu</string>
2260
+                 </property>
2261
+                </widget>
2262
+               </item>
2263
+               <item row="1" column="0">
2264
+                <widget class="QLabel" name="label_44">
2265
+                 <property name="text">
2266
+                  <string>Forgetting factor (λ)</string>
2267
+                 </property>
2268
+                </widget>
2269
+               </item>
2270
+               <item row="1" column="2">
2271
+                <widget class="QLabel" name="label_52">
2272
+                 <property name="text">
2273
+                  <string>PCA on ref</string>
2274
+                 </property>
2275
+                </widget>
2276
+               </item>
2198 2277
                <item row="0" column="0">
2199 2278
                 <widget class="QLabel" name="label_43">
2200 2279
                  <property name="text">
@@ -2215,10 +2294,19 @@ background: dark grey;
2215 2294
                  </property>
2216 2295
                 </widget>
2217 2296
                </item>
2218
-               <item row="0" column="2">
2219
-                <widget class="QLabel" name="label_51">
2297
+               <item row="4" column="3">
2298
+                <widget class="QPushButton" name="adaptGO">
2299
+                 <property name="styleSheet">
2300
+                  <string notr="true">#adaptGO {
2301
+    background: green;
2302
+}
2303
+
2304
+#adaptGO:disabled{
2305
+    background: black;
2306
+}</string>
2307
+                 </property>
2220 2308
                  <property name="text">
2221
-                  <string>Mu</string>
2309
+                  <string>GO</string>
2222 2310
                  </property>
2223 2311
                 </widget>
2224 2312
                </item>
@@ -2241,39 +2329,6 @@ background: dark grey;
2241 2329
                  </property>
2242 2330
                 </widget>
2243 2331
                </item>
2244
-               <item row="1" column="0">
2245
-                <widget class="QLabel" name="label_44">
2246
-                 <property name="text">
2247
-                  <string>Forgetting factor (λ)</string>
2248
-                 </property>
2249
-                </widget>
2250
-               </item>
2251
-               <item row="1" column="1">
2252
-                <widget class="QDoubleSpinBox" name="adaptLambdaSpinBox">
2253
-                 <property name="toolTip">
2254
-                  <string>Forgetting factor, how quickly does the filter adapt.</string>
2255
-                 </property>
2256
-                 <property name="decimals">
2257
-                  <number>3</number>
2258
-                 </property>
2259
-                 <property name="minimum">
2260
-                  <double>0.950000000000000</double>
2261
-                 </property>
2262
-                 <property name="maximum">
2263
-                  <double>1.000000000000000</double>
2264
-                 </property>
2265
-                 <property name="value">
2266
-                  <double>0.990000000000000</double>
2267
-                 </property>
2268
-                </widget>
2269
-               </item>
2270
-               <item row="1" column="2">
2271
-                <widget class="QLabel" name="label_52">
2272
-                 <property name="text">
2273
-                  <string>PCA on ref</string>
2274
-                 </property>
2275
-                </widget>
2276
-               </item>
2277 2332
                <item row="1" column="3">
2278 2333
                 <widget class="QComboBox" name="PCAComboBox">
2279 2334
                  <property name="toolTip">
@@ -2294,42 +2349,26 @@ background: dark grey;
2294 2349
                  </item>
2295 2350
                 </widget>
2296 2351
                </item>
2297
-               <item row="2" column="0">
2298
-                <widget class="QLabel" name="label_46">
2299
-                 <property name="text">
2300
-                  <string>Truncate [ms]</string>
2352
+               <item row="3" column="0" colspan="4">
2353
+                <widget class="Line" name="line_15">
2354
+                 <property name="orientation">
2355
+                  <enum>Qt::Horizontal</enum>
2301 2356
                  </property>
2302 2357
                 </widget>
2303 2358
                </item>
2304
-               <item row="2" column="1">
2305
-                <widget class="QDoubleSpinBox" name="adaptTruncateSpinBox">
2306
-                 <property name="toolTip">
2307
-                  <string>&lt;html&gt;&lt;head/&gt;&lt;body&gt;&lt;p&gt;This filter is a time-domain filter that takes some time to get going. Time-domain filters do a better job compared to frequency-domain filters in the presence of non-stationary noise. &lt;/p&gt;&lt;p&gt;The filter is run backwards, so often the late times will not be cancelled as well. You may trim records off the back using this input. &lt;/p&gt;&lt;/body&gt;&lt;/html&gt;</string>
2308
-                 </property>
2309
-                 <property name="whatsThis">
2310
-                  <string>&lt;html&gt;&lt;head/&gt;&lt;body&gt;&lt;p&gt;This filter is a time-domain filter that takes some time to get going. Time-domain filters do a better job compared to frequency-domain filters in the presence of non-stationary noise.  &lt;/p&gt;&lt;p&gt;The filter is run backwards, so often the late times will not be cancelled as well. You may trim records off the back using this input. &lt;/p&gt;&lt;/body&gt;&lt;/html&gt;</string>
2311
-                 </property>
2312
-                 <property name="maximum">
2313
-                  <double>1000.000000000000000</double>
2359
+               <item row="4" column="1">
2360
+                <widget class="QCheckBox" name="plotRLS">
2361
+                 <property name="enabled">
2362
+                  <bool>false</bool>
2314 2363
                  </property>
2315
-                 <property name="value">
2316
-                  <double>800.000000000000000</double>
2364
+                 <property name="text">
2365
+                  <string>Plot</string>
2317 2366
                  </property>
2318
-                </widget>
2319
-               </item>
2320
-               <item row="2" column="3">
2321
-                <widget class="QPushButton" name="adaptGO">
2322
-                 <property name="styleSheet">
2323
-                  <string notr="true">#adaptGO {
2324
-    background: green;
2325
-}
2326
-
2327
-#adaptGO:disabled{
2328
-    background: black;
2329
-}</string>
2367
+                 <property name="checkable">
2368
+                  <bool>true</bool>
2330 2369
                  </property>
2331
-                 <property name="text">
2332
-                  <string>GO</string>
2370
+                 <property name="checked">
2371
+                  <bool>true</bool>
2333 2372
                  </property>
2334 2373
                 </widget>
2335 2374
                </item>
@@ -4143,8 +4182,8 @@ background: dark grey;
4143 4182
               <rect>
4144 4183
                <x>0</x>
4145 4184
                <y>0</y>
4146
-               <width>100</width>
4147
-               <height>30</height>
4185
+               <width>96</width>
4186
+               <height>26</height>
4148 4187
               </rect>
4149 4188
              </property>
4150 4189
              <attribute name="label">
@@ -4525,6 +4564,7 @@ p, li { white-space: pre-wrap; }
4525 4564
      <string>File</string>
4526 4565
     </property>
4527 4566
     <addaction name="actionOpen_GMR"/>
4567
+    <addaction name="actionLoad_MIDI"/>
4528 4568
     <addaction name="separator"/>
4529 4569
     <addaction name="actionOpen_Preprocessed_Dataset"/>
4530 4570
     <addaction name="separator"/>
@@ -4614,6 +4654,11 @@ p, li { white-space: pre-wrap; }
4614 4654
     <string>Inversion</string>
4615 4655
    </property>
4616 4656
   </action>
4657
+  <action name="actionLoad_MIDI">
4658
+   <property name="text">
4659
+    <string>Load MIDI 2 Data </string>
4660
+   </property>
4661
+  </action>
4617 4662
  </widget>
4618 4663
  <customwidgets>
4619 4664
   <customwidget>

+ 138
- 0
akvo/tressel/calcAkvoKernel--save.py View File

@@ -0,0 +1,138 @@
1
+
2
+import os, sys
3
+import numpy as np
4
+from ruamel import yaml
5
+
6
+import pyLemma.LemmaCore as lc 
7
+import pyLemma.Merlin as mrln 
8
+import pyLemma.FDEM1D as em1d 
9
+
10
+import numpy as np
11
+
12
+#import matplotlib.pyplot as plt 
13
+#import seaborn as sns
14
+#sns.set(style="ticks")
15
+#import cmocean 
16
+#from SEGPlot import *
17
+#from matplotlib.ticker import FormatStrFormatter
18
+#import matplotlib.ticker as plticker
19
+
20
+
21
+# Converts Lemma/Merlin/Akvo serialized Eigen arrays into numpy ones for use by Python 
22
+class VectorXr(yaml.YAMLObject):
23
+    """
24
+    Converts Lemma/Merlin/Akvo serialized Eigen arrays into numpy ones for use by Python 
25
+    """
26
+    yaml_tag = u'VectorXr'
27
+    def __init__(self, array):
28
+        self.size = np.shape(array)[0]
29
+        self.data = array.tolist()
30
+    def __repr__(self):
31
+        # Converts to numpy array on import 
32
+        return "np.array(%r)" % (self.data)
33
+
34
+class AkvoData(yaml.YAMLObject):
35
+    """
36
+    Reads an Akvo serialized dataset into a standard python dictionary 
37
+    """
38
+    yaml_tag = u'AkvoData'
39
+    def __init__(self, array):
40
+        pass
41
+        #self.size = np.shape(array)[0]
42
+        #self.Imp = array.tolist()
43
+    def __repr__(self):
44
+        # Converts to a dictionary with Eigen vectors represented as Numpy arrays 
45
+        return self
46
+
47
+def loadAkvoData(fnamein):
48
+    """ Loads data from an Akvo YAML file. The 0.02 is hard coded as the pulse length. This needs to be 
49
+        corrected in future kernel calculations. The current was reported but not the pulse length. 
50
+    """
51
+    fname = (os.path.splitext(fnamein)[0])
52
+    with open(fnamein, 'r') as stream:
53
+        try:
54
+            AKVO = (yaml.load(stream, Loader=yaml.Loader))
55
+        except yaml.YAMLError as exc:
56
+            print(exc)
57
+    return AKVO 
58
+
59
+
60
+def main():
61
+
62
+    if len(sys.argv) < 3:
63
+        print ("usage  python calcAkvoKernel.py   AkvoDataset.yaml  Coil1.yaml  " )
64
+        exit()
65
+
66
+    AKVO = loadAkvoData(sys.argv[1])
67
+
68
+    B_inc = AKVO.META["B_0"]["inc"]  
69
+    B_dec = AKVO.META["B_0"]["dec"]  
70
+    B0 = AKVO.META["B_0"]["intensity"]  
71
+
72
+    gamma = 2.67518e8
73
+    fT = AKVO.transFreq
74
+    #B0 = (fL*2.*np.pi) /gamma * 1e9
75
+ 
76
+    Coil1 = em1d.PolygonalWireAntenna.DeSerialize( sys.argv[2] )
77
+    Coil1.SetNumberOfFrequencies(1)
78
+    Coil1.SetFrequency(0, fT) 
79
+    Coil1.SetCurrent(1.)
80
+
81
+    lmod = em1d.LayeredEarthEM() 
82
+    lmod.SetNumberOfLayers(4)
83
+    lmod.SetLayerThickness([15.49, 28.18])
84
+    lmod.SetLayerConductivity([0.0, 1./16.91, 1./24.06, 1./33.23])
85
+
86
+    lmod.SetMagneticFieldIncDecMag( B_inc, B_dec, B0, lc.NANOTESLA )
87
+    
88
+    exit()
89
+
90
+    Kern = mrln.KernelV0()
91
+    Kern.PushCoil( "Coil 1", Coil1 )
92
+    Kern.SetLayeredEarthEM( lmod );
93
+    Kern.SetIntegrationSize( (200,200,200) )
94
+    Kern.SetIntegrationOrigin( (0,0,0) )
95
+    Kern.SetTolerance( 1e-9 )
96
+    Kern.SetMinLevel( 3 )
97
+    Kern.SetHankelTransformType( lc.FHTKEY201 )
98
+    Kern.AlignWithAkvoDataset( sys.argv[1] )
99
+
100
+    thick = np.geomspace(.5, 10,num=40)
101
+    iface = np.cumsum(thick)
102
+    Kern.SetDepthLayerInterfaces(iface)
103
+    #Kern.SetDepthLayerInterfaces(np.geomspace(1, 110, num=40))
104
+    #Kern.SetDepthLayerInterfaces(np.linspace(1, 110, num=50))
105
+    #Kern.SetDepthLayerInterfaces(np.geomspace(1, 110, num=40))
106
+ 
107
+    # autAkvoDataNode = YAML::LoadFile(argv[4]);
108
+    # Kern->AlignWithAkvoDataset( AkvoDataNode );
109
+
110
+    #Kern.SetPulseDuration(0.040)
111
+    #Kern.SetPulseCurrent(  [1.6108818092452406, 1.7549935078885168, 1.7666319459646016, 1.9270787752430283,
112
+    #    1.9455431806179229, 2.111931346726564, 2.1466747256211747, 2.3218217392379588,
113
+    #    2.358359967649008, 2.5495654202189058, 2.5957289164577992, 2.8168532605800802,
114
+    #    2.85505242699187, 3.1599429539069774, 3.2263673040205068, 3.6334182368296544,
115
+    #    3.827985200119751, 4.265671313014058, 4.582237014873297, 5.116839616183394,
116
+    #    5.515173073160611, 6.143620383280934, 6.647972282096122, 7.392577402979211,
117
+    #    8.020737177449933, 8.904435233295793, 9.701975105606063, 10.74508217792577,
118
+    #    11.743887525923592, 12.995985956061467, 14.23723766879807, 15.733870137824457,
119
+    #    17.290155933625808, 19.07016662950366, 21.013341340455703, 23.134181634845618,
120
+    #    25.570925414182238, 28.100862178905476, 31.13848909847073, 34.16791099558486,
121
+    #    37.95775984680512, 41.589619321873165, 46.327607251605286, 50.667786337299205,
122
+    #    56.60102493062895, 61.81174065797068, 69.23049946198458, 75.47409803238031,
123
+    #    84.71658869065816, 92.1855007134236, 103.77129947551164, 112.84577430578537,
124
+    #    127.55127257092909, 138.70199812969176, 157.7443764728878, 171.39653462998626]
125
+    #)
126
+
127
+    Kern.CalculateK0( ["Coil 1"], ["Coil 1"], False )
128
+
129
+    yml = open('akvoK3-' + str(Kern.GetTolerance()) + '.yaml', 'w')
130
+    print(Kern, file=yml)
131
+
132
+    K0 = Kern.GetKernel()
133
+    
134
+    #plt.matshow(np.abs(K0))
135
+    #plt.show()
136
+
137
+if __name__ == "__main__":
138
+    main()

+ 729
- 0
akvo/tressel/decay-old.py View File

@@ -0,0 +1,729 @@
1
+import numpy, array #,rpy2
2
+from matplotlib import pyplot as plt
3
+import numpy as np
4
+from scipy.optimize import least_squares
5
+from rpy2.robjects.packages import importr
6
+
7
+import rpy2.robjects as robjects
8
+import rpy2.robjects.numpy2ri
9
+
10
+#import notch
11
+from numpy.fft import fft, fftfreq
12
+
13
+# We know/can calculate frequency peak, use this to guess where picks will be.
14
+# maybe have a sliding window that reports peak values.
15
+def peakPicker(data, omega, dt):
16
+
17
+    # compute window based on omega and dt
18
+    # make sure you are not aliased, grab every other peak
19
+    window = (2*numpy.pi) / (omega*dt)
20
+
21
+    data = numpy.array(data) 
22
+    peaks = []
23
+    troughs = []
24
+    times = []
25
+    times2 = []
26
+    indices = []
27
+    ws = 0
28
+    we = window
29
+    ii = 0
30
+    for i in range((int)(len(data)/window)):
31
+        
32
+        # initially was just returning this I think avg is better
33
+        #times.append( (ws + numpy.abs(data[ws:we]).argmax()) * dt )
34
+    
35
+        peaks.append(numpy.max(data[ws:we]))
36
+        times.append( (ws + data[ws:we].argmax()) * dt )
37
+        indices.append( ii + data[ws:we].argmax() )        
38
+
39
+        troughs.append(numpy.min(data[ws:we]))
40
+        times2.append( (ws + (data[ws:we]).argmin()) * dt )
41
+        indices.append( ii + data[ws:we].argmin() )        
42
+
43
+        ws += window
44
+        we += window
45
+        ii += (int)(we-ws)
46
+    
47
+    #return numpy.array(peaks), numpy.array(times)
48
+    
49
+    # Averaging peaks does a good job of removing bias in noise
50
+    return (numpy.array(peaks)-numpy.array(troughs))/2., \
51
+        (numpy.array(times)+numpy.array(times2))/2., \
52
+        indices           
53
+
54
+
55
+#################################################
56
+# Regress for T2 using rpy2 interface
57
+def regressCurve(peaks,times,sigma2=1,intercept=True):
58
+
59
+    # TODO, if regression fails, it might be because there is no exponential
60
+    # term, maybe do a second regression then on a linear model. 
61
+    b1  = 0                  # Bias
62
+    b2  = 0                  # Linear 
63
+    rT2 = 0.3                # T2 regressed
64
+    r   = robjects.r         
65
+
66
+    # Variable shared between R and Python
67
+    robjects.globalenv['b1'] = b1
68
+    robjects.globalenv['b2'] = b2
69
+    robjects.globalenv['rT2'] = rT2
70
+    robjects.globalenv['sigma2'] = sigma2
71
+    value = robjects.FloatVector(peaks)
72
+    times = robjects.FloatVector(numpy.array(times))
73
+    
74
+#    my_weights = robjects.RVector(value/sigma2)
75
+#    robjects.globalenv['my_weigts'] = my_weights
76
+
77
+#    if sigma2 != 0:
78
+#        print "weighting"
79
+#        tw = numpy.array(peaks)/sigma2 
80
+#        my_weights = robjects.RVector( tw/numpy.max(tw) )
81
+#    else:
82
+#        my_weights = robjects.RVector(numpy.ones(len(peaks))) 
83
+
84
+#    robjects.globalenv['my_weights'] = my_weights
85
+    
86
+    if (intercept):
87
+        my_list = robjects.r('list(b1=50, b2=1e2, rT2=0.03)')
88
+        my_lower = robjects.r('list(b1=0, b2=0, rT2=.005)')
89
+        my_upper = robjects.r('list(b1=20000, b2=2000, rT2=.700)')
90
+    else:
91
+        my_list = robjects.r('list(b2=1e2, rT2=0.3)')
92
+        my_lower = robjects.r('list(b2=0, rT2=.005)')
93
+        my_upper = robjects.r('list(b2=2000, rT2=.700)')
94
+
95
+    my_cont = robjects.r('nls.control(maxiter=1000, warnOnly=TRUE, printEval=FALSE)')
96
+
97
+    
98
+    if (intercept):
99
+        #fmla = robjects.RFormula('value ~ b1 + exp(-times/rT2)')
100
+        fmla = robjects.Formula('value ~ b1 + b2*exp(-times/rT2)')
101
+        #fmla = robjects.RFormula('value ~ b1 + b2*times + exp(-times/rT2)')
102
+    else:
103
+        fmla = robjects.Formula('value ~ b2*exp(-times/rT2)')
104
+
105
+    env = fmla.getenvironment()
106
+    env['value'] = value
107
+    env['times'] = times
108
+    
109
+    # ugly, but I get errors with everything else I've tried
110
+    my_weights = robjects.r('rep(1,length(value))')
111
+    for ii in range(len(my_weights)):
112
+        my_weights[ii] *= peaks[ii]/sigma2
113
+    Error = False
114
+    #fit = robjects.r.nls(fmla,start=my_list,control=my_cont,weights=my_weights)
115
+    if (sigma2 != 1):
116
+        print("SIGMA 2")
117
+        #fit = robjects.r.tryCatch(robjects.r.suppressWarnings(robjects.r.nls(fmla,start=my_list,control=my_cont,algorithm="port", \
118
+        #                     weights=my_weights)), 'silent=TRUE')
119
+        fit = robjects.r.tryCatch(robjects.r.nls(fmla,start=my_list,control=my_cont))#, \
120
+                            # weights=my_weights))
121
+    else:
122
+        try:
123
+            fit = robjects.r.tryCatch(robjects.r.nls(fmla,start=my_list,control=my_cont,algorithm="port"))#,lower=my_lower,upper=my_upper))
124
+        except:
125
+            print("regression issue pass")
126
+            Error = True
127
+    # If failure fall back on zero regression values   
128
+    if not Error:
129
+        #Error = fit[3][0]
130
+        report =  r.summary(fit)
131
+    b1 = 0
132
+    b2 = 0 
133
+    rT2 = 1
134
+    if (intercept):
135
+        if not Error:
136
+            b1  =  r['$'](report,'par')[0]
137
+            b2  =  r['$'](report,'par')[1]
138
+            rT2 =  r['$'](report,'par')[2]
139
+            #print  report
140
+            #print  r['$'](report,'convergence')
141
+            #print  r['convergence'] #(report,'convergence')
142
+            #print  r['$'](report,'par')[13]
143
+            #print  r['$'](report,'par')[14]
144
+        else:
145
+            print("ERROR DETECTED, regressed values set to default")
146
+            b1 = 1e1
147
+            b2 = 1e-2
148
+            rT2 = 1e-2
149
+            #print r['$'](report,'par')[0]
150
+            #print r['$'](report,'par')[1]
151
+            #print r['$'](report,'par')[2]
152
+        return [b1,b2,rT2] 
153
+    else:
154
+        if not Error:
155
+            rT2 =  r['$'](report,'par')[1]
156
+            b2  =  r['$'](report,'par')[0]
157
+        else:
158
+            print("ERROR DETECTED, regressed values set to default")
159
+        return [b2, rT2] 
160
+
161
+#################################################
162
+# Regress for T2 using rpy2 interface
163
+def regressCurve2(peaks,times,sigma2=[None],intercept=True):
164
+
165
+    if sigma2[0] != None:
166
+        my_weights = robjects.FloatVector( sigma2 )
167
+
168
+    # TODO, if regression fails, it might be because there is no exponential
169
+    # term, maybe do a second regression then on a linear model. 
170
+    b1  = 0                  # Bias
171
+    b2  = 0                  # Linear 
172
+    bb2  = 0                 # Linear 
173
+    rT2 = 0.3                # T2 regressed
174
+    rrT2 = 1.3               # T2 regressed
175
+    r   = robjects.r         
176
+
177
+    # Variable shared between R and Python
178
+    robjects.globalenv['b1'] = b1
179
+    robjects.globalenv['b2'] = b2
180
+    robjects.globalenv['rT2'] = rT2
181
+    
182
+    robjects.globalenv['bb2'] = b2
183
+    robjects.globalenv['rrT2'] = rT2
184
+    
185
+    #robjects.globalenv['sigma2'] = sigma2
186
+    value = robjects.FloatVector(peaks)
187
+    times = robjects.FloatVector(numpy.array(times))
188
+    
189
+    
190
+    if (intercept):
191
+        my_list = robjects.r('list(b1=.50, b2=1e2, rT2=0.03, bb2=1e1, rrT2=1.3)')
192
+        my_lower = robjects.r('list(b1=0, b2=0, rT2=.005, bb2=0, rrT2=.005 )')
193
+        my_upper = robjects.r('list(b1=2000, b2=2000, rT2=.700, bb2=2000, rrT2=1.3 )')
194
+    else:
195
+        my_list  = robjects.r('list(b2=.5, rT2=0.3,  bb2=.5, rrT2=1.3)')
196
+        my_lower = robjects.r('list(b2=0,  rT2=.005, bb2=0,  rrT2=.005)')
197
+        my_upper = robjects.r('list(b2=1,  rT2=2.6,    bb2=1,  rrT2=2.6)')
198
+
199
+    my_cont = robjects.r('nls.control(maxiter=1000, warnOnly=TRUE, printEval=FALSE)')
200
+
201
+    
202
+    if (intercept):
203
+        #fmla = robjects.RFormula('value ~ b1 + exp(-times/rT2)')
204
+        fmla = robjects.Formula('value ~ b1 + b2*exp(-times/rT2) + bb2*exp(-times/rrT2)')
205
+        #fmla = robjects.RFormula('value ~ b1 + b2*times + exp(-times/rT2)')
206
+    else:
207
+        fmla = robjects.Formula('value ~ b2*exp(-times/rT2) + bb2*exp(-times/rrT2)')
208
+
209
+    env = fmla.getenvironment()
210
+    env['value'] = value
211
+    env['times'] = times
212
+    
213
+    # ugly, but I get errors with everything else I've tried
214
+    Error = False
215
+    #fit = robjects.r.nls(fmla,start=my_list,control=my_cont,weights=my_weights)
216
+    if (sigma2[0] != None):
217
+        #print("SIGMA 2")
218
+        #fit = robjects.r.tryCatch(robjects.r.suppressWarnings(robjects.r.nls(fmla,start=my_list,control=my_cont,algorithm="port", \
219
+        #                     weights=my_weights)), 'silent=TRUE')
220
+        fit = robjects.r.tryCatch(robjects.r.nls(fmla,start=my_list,control=my_cont,algorithm='port',weights=my_weights,lower=my_lower,upper=my_upper))#, \
221
+                            # weights=my_weights))
222
+    else:
223
+        try:
224
+            fit = robjects.r.tryCatch(robjects.r.nls(fmla,start=my_list,control=my_cont,algorithm="port"))#,lower=my_lower,upper=my_upper))
225
+        except:
226
+            print("regression issue pass")
227
+            Error = True
228
+    # If failure fall back on zero regression values   
229
+    if not Error:
230
+        #Error = fit[3][0]
231
+        report =  r.summary(fit)
232
+    b1 = 0
233
+    b2 = 0 
234
+    rT2 = 1
235
+    if (intercept):
236
+        if not Error:
237
+            b1  =  r['$'](report,'par')[0]
238
+            b2  =  r['$'](report,'par')[1]
239
+            rT2 =  r['$'](report,'par')[2]
240
+            #print  report
241
+            #print  r['$'](report,'convergence')
242
+            #print  r['convergence'] #(report,'convergence')
243
+            #print  r['$'](report,'par')[13]
244
+            #print  r['$'](report,'par')[14]
245
+        else:
246
+            print("ERROR DETECTED, regressed values set to default")
247
+            b1 = 1e1
248
+            b2 = 1e-2
249
+            rT2 = 1e-2
250
+            #print r['$'](report,'par')[0]
251
+            #print r['$'](report,'par')[1]
252
+            #print r['$'](report,'par')[2]
253
+        return [b1,b2,rT2, bb2, rrT2] 
254
+    else:
255
+        if not Error:
256
+            rT2 =  r['$'](report,'par')[1]
257
+            b2  =  r['$'](report,'par')[0]
258
+            rrT2 =  r['$'](report,'par')[3]
259
+            bb2  =  r['$'](report,'par')[2]
260
+        else:
261
+            print("ERROR DETECTED, regressed values set to default")
262
+        return [b2, rT2, bb2, rrT2] 
263
+
264
+def fun(x, t, y):
265
+    """ Cost function for regression, single exponential, no DC term 
266
+        x[0] = A0
267
+        x[1] = zeta 
268
+        x[2] = df
269
+        x[3] = T2
270
+    """
271
+    # concatenated real and imaginary parts  
272
+    pre =  np.concatenate((-x[0]*np.sin(2.*np.pi*x[2]*t + x[1])*np.exp(-t/x[3]), \
273
+                           +x[0]*np.cos(2.*np.pi*x[2]*t + x[1])*np.exp(-t/x[3])))  
274
+    return y-pre
275
+
276
+def fun2(x, t, y):
277
+    """ Cost function for regression, single exponential, no DC term 
278
+        x[0] = A0
279
+        x[1] = zeta 
280
+        x[2] = T2
281
+    """
282
+    # concatenated real and imaginary parts  
283
+    pre =  np.concatenate((x[0]*np.cos(x[1])*np.exp(-t/x[2]), \
284
+                       -1.*x[0]*np.sin(x[1])*np.exp(-t/x[2])))  
285
+    return y-pre
286
+
287
+
288
+def quadratureDetect2(X, Y, tt, x0="None"): 
289
+    """ Pure python quadrature detection using Scipy.  
290
+        X = real part of NMR signal 
291
+        Y = imaginary component of NMR signal 
292
+        tt = time 
293
+    """
294
+    print("Pure Python Quad Det", "TODO look at loss functions and method")
295
+    # Loss functions, linear, soft_l1, huber, cauchy, arctan 
296
+    # df
297
+    loss = 'cauchy'  #  'soft_l1'
298
+    method = 'trf'   # trf, dogbox, lm 
299
+    if x0=="None":
300
+        x0 = np.array( [1., 0., 0., .2] ) # A0, zeta, df, T2 
301
+        res_lsq = least_squares(fun, x0, args=(tt, np.concatenate((X, Y))), loss=loss, f_scale=1.0,\
302
+            bounds=( [1., -np.pi, -5, .005] , [1000., np.pi, 5, .800] ),
303
+            method=method 
304
+            )
305
+        x = res_lsq.x 
306
+        print ("df", x[0], x[1], x[2], x[3])
307
+    else:
308
+        res_lsq = least_squares(fun, x0, args=(tt, np.concatenate((X, Y))), loss=loss, f_scale=1.0,\
309
+            bounds=( [1., -np.pi, -5, .005] , [1000., np.pi, 5, .800] ),
310
+            method=method 
311
+            )
312
+
313
+        #bounds=( [0., 0, -20, .0] , [1., np.pi, 20, .6] ))
314
+
315
+    x = res_lsq.x 
316
+    return res_lsq.success, x[0], x[2], x[1], x[3]
317
+    
318
+    # no df
319
+    #x = np.array( [1., 0., 0.2] )
320
+    #res_lsq = least_squares(fun2, x, args=(tt, np.concatenate((X, Y))), loss='soft_l1', f_scale=0.1)
321
+    #x = res_lsq.x 
322
+    #return conv, E0,df,phi,T2
323
+    #return res_lsq.success, x[0], 0, x[1], x[2]
324
+
325
+def quadratureDetect(X, Y, tt, CorrectFreq=False, BiExp=False, CorrectDC=False):
326
+ 
327
+    r   = robjects.r        
328
+
329
+    if CorrectDC:
330
+        robjects.r(''' 
331
+             Xc1 <- function(E01, df, tt, phi, T2_1, DC) {
332
+	                DC + E01*cos(2*pi*df*tt + phi) * exp(-tt/T2_1)
333
+            }
334
+    
335
+            Yc1 <- function(E01, df, tt, phi, T2_1, DC) {
336
+	                DC - E01*sin(2*pi*df*tt + phi) * exp(-tt/T2_1)
337
+            } 
338
+            ''')
339
+    else:   
340
+        robjects.r(''' 
341
+             Xc1 <- function(E01, df, tt, phi, T2_1) {
342
+	                E01*cos(2*pi*df*tt + phi) * exp(-tt/T2_1)
343
+            }
344
+    
345
+            Yc1 <- function(E01, df, tt, phi, T2_1) {
346
+	                -E01*sin(2*pi*df*tt + phi) * exp(-tt/T2_1)
347
+            } 
348
+            ''')
349
+
350
+    # bi-exponential 
351
+    if CorrectDC:
352
+        robjects.r(''' 
353
+             Xc2 <- function(E01, E02, df, tt, phi, T2_1, T2_2, DC) {
354
+	               DC + E01*cos(2*pi*df*tt + phi) * exp(-tt/T2_1) + 
355
+	                DC + E02*cos(2*pi*df*tt + phi) * exp(-tt/T2_2)
356
+            }
357
+
358
+            Yc2 <- function(E01, E02, df, tt, phi, T2_1, T2_2, DC) {
359
+	                DC - E01*sin(2*pi*df*tt + phi) * exp(-tt/T2_1) + 
360
+	                DC - E02*sin(2*pi*df*tt + phi) * exp(-tt/T2_2)
361
+            } 
362
+            ''')
363
+    else:   
364
+        robjects.r(''' 
365
+             Xc2 <- function(E01, E02, df, tt, phi, T2_1, T2_2) {
366
+	               E01*cos(2*pi*df*tt + phi) * exp(-tt/T2_1) + 
367
+	               E02*cos(2*pi*df*tt + phi) * exp(-tt/T2_2)
368
+            }
369
+
370
+            Yc2 <- function(E01, E02, df, tt, phi, T2_1, T2_2) {
371
+	                -E01*sin(2*pi*df*tt + phi) * exp(-tt/T2_1) + 
372
+	                -E02*sin(2*pi*df*tt + phi) * exp(-tt/T2_2)
373
+            } 
374
+            ''')
375
+
376
+    # Make 0 vector 
377
+    Zero = robjects.FloatVector(numpy.zeros(len(X)))
378
+    
379
+    # Fitted Parameters
380
+    E01 = 0.
381
+    E02 = 0.
382
+    df = 0.
383
+    phi = 0.
384
+    T2_1 = 0.
385
+    T2_2 = 0.
386
+    DC = 0.
387
+    robjects.globalenv['DC'] = DC
388
+    robjects.globalenv['E01'] = E01
389
+    robjects.globalenv['E02'] = E02
390
+    robjects.globalenv['df'] = df
391
+    robjects.globalenv['phi'] = phi
392
+    robjects.globalenv['T2_1'] = T2_1
393
+    robjects.globalenv['T2_2'] = T2_2
394
+    XY = robjects.FloatVector(numpy.concatenate((X,Y)))
395
+    
396
+    # Arrays
397
+    tt = robjects.FloatVector(numpy.array(tt))
398
+    X = robjects.FloatVector(numpy.array(X))
399
+    Y = robjects.FloatVector(numpy.array(Y))
400
+    Zero = robjects.FloatVector(numpy.array(Zero))
401
+
402
+    
403
+
404
+    if BiExp:
405
+        if CorrectDC:
406
+            fmla = robjects.Formula('XY ~ c(Xc2( E01, E02, df, tt, phi, T2_1, T2_2, DC ), Yc2( E01, E02, df, tt, phi, T2_1, T2_2, DC ))')
407
+            if CorrectFreq:    
408
+                start = robjects.r('list(E01=.100, E02=.01,   df=0,    phi=0.    ,  T2_1=.100, T2_2=.01, DC=0.0)')
409
+                lower = robjects.r('list(E01=1e-6, E02=1e-6,  df=-50,  phi=-3.14 ,  T2_1=.001, T2_2=.001, DC=0.0)')
410
+                upper = robjects.r('list(E01=1.00, E02=1.0,   df=50,   phi=3.14  ,  T2_1=.800, T2_2=.8, DC=0.5)')
411
+            else:
412
+                start = robjects.r('list(E01=.100, E02=.01,   phi=0.9   ,  T2_1=.100, T2_2=.01,  DC=0.0)')
413
+                lower = robjects.r('list(E01=1e-6, E02=1e-6,  phi=-3.14 ,  T2_1=.001, T2_2=.001, DC=0.0)')
414
+                upper = robjects.r('list(E01=1.00, E02=1.0,   phi=3.14  ,  T2_1=.800, T2_2=.8,   DC=0.5)')
415
+        else:
416
+            fmla = robjects.Formula('XY ~ c(Xc2( E01, E02, df, tt, phi, T2_1, T2_2 ), Yc2( E01, E02, df, tt, phi, T2_1, T2_2))')
417
+            if CorrectFreq:    
418
+                start = robjects.r('list(E01=.100, E02=.01,   df=0,    phi=0.    ,  T2_1=.100, T2_2=.01)')
419
+                lower = robjects.r('list(E01=1e-6, E02=1e-6,  df=-50,  phi=-3.14 ,  T2_1=.001, T2_2=.001)')
420
+                upper = robjects.r('list(E01=1.00, E02=1.0,   df=50,   phi=3.14  ,  T2_1=.800, T2_2=.8)')
421
+            else:
422
+                start = robjects.r('list(E01=.100, E02=.01,   phi=0.9   ,  T2_1=.100, T2_2=.01)')
423
+                lower = robjects.r('list(E01=1e-6, E02=1e-6,  phi=-3.14 ,  T2_1=.001, T2_2=.001)')
424
+                upper = robjects.r('list(E01=1.00, E02=1.0,   phi=3.14  ,  T2_1=.800, T2_2=.8)')
425
+    else: 
426
+        if CorrectDC:
427
+            fmla = robjects.Formula('XY ~ c(Xc1( E01, df, tt, phi, T2_1, DC), Yc1( E01, df, tt, phi, T2_1,DC))')
428
+            if CorrectFreq:    
429
+                start = robjects.r('list(E01=.100, df=0   , phi=0.   , T2_1=.100, DC=0.0)')
430
+                lower = robjects.r('list(E01=1e-6, df=-50., phi=-3.14, T2_1=.001, DC=0.0)')
431
+                upper = robjects.r('list(E01=1.00, df=50. , phi=3.14 , T2_1=.800, DC=0.5)')
432
+            else:
433
+                start = robjects.r('list(E01=.100, phi= 0.  , T2_1=.100, DC=0.0)')
434
+                lower = robjects.r('list(E01=1e-6, phi=-3.13, T2_1=.001, DC=0.0)')
435
+                upper = robjects.r('list(E01=1.00, phi= 3.13, T2_1=.800, DC=0.5)')
436
+        else:
437
+            fmla = robjects.Formula('XY ~ c(Xc1( E01, df, tt, phi, T2_1), Yc1( E01, df, tt, phi, T2_1))')
438
+            if CorrectFreq:    
439
+                start = robjects.r('list(E01=.100, df=0     , phi=0.   ,  T2_1=.100)')
440
+                lower = robjects.r('list(E01=1e-6, df=-50. , phi=-3.14 ,  T2_1=.001)')
441
+                upper = robjects.r('list(E01=1.00, df=50.  , phi=3.14  ,  T2_1=.800)')
442
+            else:
443
+                start = robjects.r('list(E01=.100, phi= 0.  , T2_1=.100)')
444
+                lower = robjects.r('list(E01=1e-6, phi=-3.13, T2_1=.001)')
445
+                upper = robjects.r('list(E01=1.00, phi= 3.13, T2_1=.800)')
446
+
447
+    env = fmla.getenvironment()
448
+    env['Zero'] = Zero
449
+    env['X'] = X
450
+    env['Y'] = Y
451
+    env['XY'] = XY 
452
+    env['tt'] = tt
453
+
454
+    cont = robjects.r('nls.control(maxiter=10000, warnOnly=TRUE, printEval=FALSE)')
455
+    
456
+    fit = robjects.r.tryCatch(robjects.r.nls(fmla, start=start, control=cont, lower=lower, upper=upper, algorithm='port')) #, \
457
+    #fit = robjects.r.tryCatch(robjects.r.nls(fmla, start=start, control=cont)) #, \
458
+    report =  r.summary(fit)
459
+
460
+    conv = r['$'](fit,'convergence')[0]
461
+    #if conv:
462
+    #    print (report)
463
+    #    print ("conv", conv)
464
+    print ("Conv",  r['$'](fit,'convergence'))  # T2
465
+    print (report)
466
+    
467
+    if BiExp:
468
+        if CorrectFreq:    
469
+            E0   =  r['$'](report,'par')[0]   # E01
470
+            E0  +=  r['$'](report,'par')[1]   # E02
471
+            df  =  r['$'](report,'par')[2]   # offset
472
+            phi =  r['$'](report,'par')[3]   # phase 
473
+            T2  =  r['$'](report,'par')[4]   # T2
474
+        else:
475
+            E0   =  r['$'](report,'par')[0]   # E01
476
+            E0  +=  r['$'](report,'par')[1]   # E02
477
+            phi =  r['$'](report,'par')[2]   # phase 
478
+            T2  =  r['$'](report,'par')[3]   # T2
479
+    else:
480
+        if CorrectFreq:    
481
+            E0   =  r['$'](report,'par')[0]   # E01
482
+            df  =  r['$'](report,'par')[1]   # offset
483
+            phi =  r['$'](report,'par')[2]   # phase 
484
+            T2  =  r['$'](report,'par')[3]   # T2
485
+        else:
486
+            E0   =  r['$'](report,'par')[0]   # E01
487
+            phi =  r['$'](report,'par')[1]   # phase 
488
+            T2  =  r['$'](report,'par')[2]   # T2
489
+    #phi = 0.907655876627
490
+    #phi = 0
491
+    #print ("df", df)# = 0
492
+    return conv, E0,df,phi,T2
493
+    
494
+
495
+#################################################
496
+# Regress for T2 using rpy2 interface
497
+def regressSpec(w, wL, X): #,sigma2=1,intercept=True):
498
+
499
+    # compute s
500
+    s = -1j*w
501
+
502
+    # TODO, if regression fails, it might be because there is no exponential
503
+    # term, maybe do a second regression then on a linear model. 
504
+    a   = 0                  # Linear 
505
+    rT2 = 0.1                # T2 regressed
506
+    r   = robjects.r         
507
+
508
+    # Variable shared between R and Python
509
+    robjects.globalenv['a'] = a
510
+    robjects.globalenv['rT2'] = rT2
511
+    robjects.globalenv['wL'] = wL
512
+    robjects.globalenv['nb'] = 0
513
+
514
+    s = robjects.ComplexVector(numpy.array(s))
515
+    XX = robjects.ComplexVector(X)
516
+    Xr = robjects.FloatVector(numpy.real(X))
517
+    Xi = robjects.FloatVector(numpy.imag(X))
518
+    Xa = robjects.FloatVector(numpy.abs(X))
519
+    Xri = robjects.FloatVector(numpy.concatenate((Xr,Xi)))
520
+    
521
+    #my_lower = robjects.r('list(a=.001, rT2=.001, nb=.0001)')
522
+    my_lower = robjects.r('list(a=.001, rT2=.001)')
523
+    #my_upper = robjects.r('list(a=1.5, rT2=.300, nb =100.)')
524
+    my_upper = robjects.r('list(a=1.5, rT2=.300)')
525
+     
526
+    #my_list = robjects.r('list(a=.2, rT2=0.03, nb=.1)')
527
+    my_list = robjects.r('list(a=.2, rT2=0.03)')
528
+    my_cont = robjects.r('nls.control(maxiter=5000, warnOnly=TRUE, printEval=FALSE)')
529
+    
530
+    #fmla = robjects.Formula('Xri ~ c(a*Re((wL) / (wL^2+(s+1/rT2)^2 )), a*Im((wL)/(wL^2 + (s+1/rT2)^2 )))') # envelope
531
+    ##fmla = robjects.Formula('Xri ~ c(a*Re((wL) / (wL^2+(s+1/rT2)^2 )), a*Im((wL)/(wL^2 + (s+1/rT2)^2 )))') # envelope
532
+    #fmla = robjects.Formula('XX ~ a*(wL) / (wL^2 + (s+1/rT2)^2 )') # complex
533
+    #fmla = robjects.Formula('Xa ~ abs(a*(wL) / (wL^2 + (s+1/rT2)^2 )) + nb') # complex
534
+    fmla = robjects.Formula('Xa ~ abs(a*(wL) / (wL^2 + (s+1/rT2)^2 ))') # complex
535
+ 
536
+    env = fmla.getenvironment()
537
+    env['s'] = s
538
+    env['Xr'] = Xr
539
+    env['Xa'] = Xa
540
+    env['Xi'] = Xi
541
+    env['Xri'] = Xri
542
+    env['XX'] = XX
543
+     
544
+    #fit = robjects.r.tryCatch(robjects.r.nls(fmla,start=my_list, control=my_cont)) #, lower=my_lower, algorithm='port')) #, \
545
+    fit = robjects.r.tryCatch(robjects.r.nls(fmla, start=my_list, control=my_cont, lower=my_lower, upper=my_upper, algorithm='port')) #, \
546
+    report =  r.summary(fit)
547
+    #print report 
548
+    #print  r.warnings()
549
+ 
550
+    a  =  r['$'](report,'par')[0]
551
+    rT2 =  r['$'](report,'par')[1]
552
+    nb =  r['$'](report,'par')[2]
553
+    
554
+    return a, rT2, nb
555
+
556
+#################################################
557
+# Regress for T2 using rpy2 interface
558
+def regressSpecComplex(w, wL, X): #,sigma2=1,intercept=True):
559
+
560
+    # compute s
561
+    s = -1j*w
562
+
563
+    # TODO, if regression fails, it might be because there is no exponential
564
+    # term, maybe do a second regression then on a linear model. 
565
+    a   = 1                  # Linear 
566
+    rT2 = 0.1                # T2 regressed
567
+    r   = robjects.r         
568
+    phi2 = 0                 # phase
569
+    wL2 = wL
570
+
571
+    # Variable shared between R and Python
572
+    robjects.globalenv['a'] = a
573
+    robjects.globalenv['rT2'] = rT2
574
+    robjects.globalenv['wL'] = wL
575
+    robjects.globalenv['wL2'] = 0
576
+    robjects.globalenv['nb'] = 0
577
+    robjects.globalenv['phi2'] = phi2
578
+
579
+    s = robjects.ComplexVector(numpy.array(s))
580
+    XX = robjects.ComplexVector(X)
581
+    Xr = robjects.FloatVector(numpy.real(X))
582
+    Xi = robjects.FloatVector(numpy.imag(X))
583
+    Xa = robjects.FloatVector(numpy.abs(X))
584
+    Xri = robjects.FloatVector(numpy.concatenate((X.real,X.imag)))
585
+
586
+    robjects.r(''' 
587
+        source('kernel.r')
588
+    ''')   
589
+    #Kw = robjects.globalenv['Kwri']
590
+     
591
+    #print (numpy.shape(X))
592
+    
593
+    #my_lower = robjects.r('list(a=.001, rT2=.001, nb=.0001)')
594
+    #my_lower = robjects.r('list(a=.001, rT2=.001)') # Working
595
+    my_lower = robjects.r('list(a=.001, rT2=.001, phi2=-3.14, wL2=wL-5)')
596
+    #my_upper = robjects.r('list(a=1.5, rT2=.300, nb =100.)')
597
+    my_upper = robjects.r('list(a=3.5, rT2=.300, phi2=3.14, wL2=wL+5)')
598
+     
599
+    #my_list = robjects.r('list(a=.2, rT2=0.03, nb=.1)')
600
+    my_list = robjects.r('list(a=.2, rT2=0.03, phi2=0, wL2=wL)')
601
+    my_cont = robjects.r('nls.control(maxiter=5000, warnOnly=TRUE, printEval=FALSE)')
602
+    
603
+    #fmla = robjects.Formula('Xri ~ c(a*Re((wL) / (wL^2+(s+1/rT2)^2 )), a*Im((wL)/(wL^2 + (s+1/rT2)^2 )))') # envelope
604
+    #fmla = robjects.Formula('Xi   ~   Im(a*(sin(phi2)*s + ((1/rT2)*sin(phi2)) + wL*cos(phi2)) / (wL^2+(s+1/rT2)^2 ))') # envelope
605
+    #fmla = robjects.Formula('Xri ~ c(Re(a*(sin(phi2)*s + ((1/rT2)*sin(phi2)) + wL*cos(phi2)) / (wL^2+(s+1/rT2)^2 )), Im(a*(sin(phi2)*s + ((1/rT2)*sin(phi2)) + wL*cos(phi2)) / (wL^2+(s+1/rT2)^2 )))') # envelope
606
+    
607
+    #fmlar = robjects.Formula('Xr ~ (Kwr(a, phi2, s, rT2, wL)) ') # envelope
608
+    #fmlai = robjects.Formula('Xi ~ (Kwi(a, phi2, s, rT2, wL)) ') # envelope
609
+    fmla = robjects.Formula('Xri ~ c(Kwr(a, phi2, s, rT2, wL2), Kwi(a, phi2, s, rT2, wL2) ) ') # envelope
610
+    #fmla = robjects.Formula('Xri ~ (Kwri(a, phi2, s, rT2, wL)) ') # envelope
611
+    
612
+    #fmla = robjects.Formula('Xa ~ (abs(a*(sin(phi2)*s + ((1/rT2)*sin(phi2)) + wL*cos(phi2)) / (wL^2+(s+1/rT2)^2 )))') # envelope
613
+    #fmla = robjects.Formula('XX ~ a*(wL) / (wL^2 + (s+1/rT2)^2 )') # complex
614
+    #fmla = robjects.Formula('Xa ~ abs(a*(wL) / (wL^2 + (s+1/rT2)^2 )) + nb') # complex
615
+    
616
+    #fmla = robjects.Formula('Xri ~ c(a*Re((wL) / (wL^2+(s+1/rT2)^2 )), a*Im((wL)/(wL^2 + (s+1/rT2)^2 )))') # envelope
617
+    
618
+    #        self.Gw[iw, iT2] = ((np.sin(phi2) *  (alpha + 1j*self.w[iw]) + self.wL*np.cos(phi2)) / \
619
+    #                               (self.wL**2 + (alpha+1.j*self.w[iw])**2 ))
620
+    #        self.Gw[iw, iT2] = ds * self.sc*((np.sin(phi2)*( alpha + 1j*self.w[iw]) + self.wL*np.cos(phi2)) / \
621
+    #                               (self.wL**2 + (alpha+1.j*self.w[iw])**2 ))
622
+    
623
+    # Works Amplitude Only!
624
+    #fmla = robjects.Formula('Xa ~ abs(a*(wL) / (wL^2 + (s+1/rT2)^2 ))') # complex
625
+ 
626
+    env = fmla.getenvironment()
627
+    env['s'] = s
628
+    env['Xr'] = Xr
629
+    env['Xa'] = Xa
630
+    env['Xi'] = Xi
631
+    env['Xri'] = Xri
632
+    env['XX'] = XX
633
+     
634
+    fit = robjects.r.tryCatch(robjects.r.nls(fmla,start=my_list, control=my_cont)) #, lower=my_lower, algorithm='port')) #, \
635
+    #fitr = robjects.r.tryCatch(robjects.r.nls(fmlar, start=my_list, control=my_cont, lower=my_lower, upper=my_upper, algorithm='port')) #, \
636
+    
637
+    #env = fmlai.getenvironment()
638
+    #fiti = robjects.r.tryCatch(robjects.r.nls(fmlai, start=my_list, control=my_cont, lower=my_lower, upper=my_upper, algorithm='port')) #, \
639
+    
640
+    #reportr =  r.summary(fitr)
641
+    #reporti =  r.summary(fiti)
642
+    report =  r.summary(fit)
643
+    #print( report )
644
+    #exit()
645
+    #print( reportr )
646
+    #print( reporti  )
647
+    #exit()
648
+    #print  r.warnings()
649
+ 
650
+    #a   =  (r['$'](reportr,'par')[0] + r['$'](reporti,'par')[0]) / 2.
651
+    #rT2 =  (r['$'](reportr,'par')[1] + r['$'](reporti,'par')[1]) / 2.
652
+    #nb  =  (r['$'](reportr,'par')[2] + r['$'](reporti,'par')[2]) / 2.
653
+    a   =  r['$'](report,'par')[0] 
654
+    rT2 =  r['$'](report,'par')[1] 
655
+    nb  =  r['$'](report,'par')[2] #phi2 
656
+
657
+    print ("Python wL2", r['$'](report,'par')[3] )   
658
+    print ("Python zeta", r['$'](report,'par')[2] )   
659
+ 
660
+    return a, rT2, nb
661
+
662
+
663
+
664
+###################################################################
665
+###################################################################
666
+###################################################################
667
+if __name__ == "__main__":
668
+
669
+    dt    = .0001
670
+    T2    = .1
671
+    omega = 2000.*2*numpy.pi
672
+    phi   = .0
673
+    T     = 8.*T2
674
+    
675
+    t = numpy.arange(0, T, dt)
676
+
677
+    # Synthetic data, simple single decaying sinusoid 
678
+    # with a single decay parameter and gaussian noise added 
679
+    data = numpy.exp(-t/T2) * numpy.sin(omega * t + phi) + numpy.random.normal(0,.05,len(t)) \
680
+                         + numpy.random.randint(-1,2,len(t))*numpy.random.exponential(.2,len(t)) 
681
+    cdata = numpy.exp(-t/T2) * numpy.sin(omega * t + phi) #+ numpy.random.normal(0,.25,len(t))
682
+    #data = numpy.random.normal(0,.25,len(t))
683
+
684
+    sigma2 = numpy.std(data[::-len(data)/4])
685
+    #sigma2 = numpy.var(data[::-len(data)/4])
686
+    print("sigma2", sigma2)    
687
+    
688
+    [peaks,times,indices] = peakPicker(data, omega, dt)
689
+    
690
+    [b1,b2,rT2] = regressCurve(peaks,times)
691
+    print("rT2 nonweighted", rT2)
692
+    
693
+    [b1,b2,rT2] = regressCurve(peaks,times,sigma2)
694
+    print("rT2 weighted", rT2)
695
+
696
+    envelope   =  numpy.exp(-t/T2)
697
+    renvelope  =  numpy.exp(-t/rT2)
698
+
699
+    #outf = file('regress.txt','w')
700
+    #for i in range(len(times)):
701
+    #    outf.write(str(times[i]) + "   " +  str(peaks[i]) + "\n")  
702
+    #outf.close()
703
+
704
+    plt.plot(t,data, 'b')
705
+    plt.plot(t,cdata, 'g', linewidth=1)
706
+    plt.plot(t,envelope, color='violet', linewidth=4)
707
+    plt.plot(t,renvelope, 'r', linewidth=4)
708
+    plt.plot(times, numpy.array(peaks), 'bo', markersize=8, alpha=.25)
709
+    plt.legend(['noisy data','clean data','real envelope','regressed env','picks'])
710
+    plt.savefig("regression.pdf")
711
+
712
+
713
+    # FFT check
714
+    fourier = fft(data)
715
+    plt.figure()
716
+    freq = fftfreq(len(data), d=dt)
717
+    plt.plot(freq, (fourier.real))
718
+    
719
+    plt.show()
720
+
721
+    # TODO do a bunch in batch mode to see if T2 estimate is better with or without 
722
+    # weighting and which model is best.
723
+
724
+    # TODO try with real data
725
+
726
+    # TODO test filters (median, FFT, notch)
727
+
728
+    # It looks like weighting is good for relatively low sigma, but for noisy data
729
+    # it hurts us. Check

+ 392
- 0
akvo/tressel/gateIntegrate4.py View File

@@ -0,0 +1,392 @@
1
+from __future__ import division 
2
+
3
+import matplotlib as mpl
4
+mpl.use('pdf')
5
+
6
+#from rasterize import rasterize_and_save
7
+
8
+import matplotlib.patches as mpatches
9
+from pwctime import pwcTime
10
+from logbarrier import * 
11
+from perlin import perlin
12
+from scipy import stats
13
+import cmocean
14
+import sys 
15
+import numpy as np 
16
+import seaborn as sns
17
+
18
+def bootstrapWindows(N, nboot, isum, adapt=False):
19
+    """ Bootstraps noise as a function of gate width
20
+        N = input noise signal 
21
+        nboot = number of boostrap windows to perform 
22
+        isum = length of windows (L_i)
23
+        adapt = reduce nboot as window size increases
24
+    """
25
+    nc = np.shape(N)[0]
26
+    Means = {}
27
+
28
+    if adapt:
29
+        Means = -9999*np.ones((len(isum), nboot//isum[0])) # dummy value
30
+        for ii, nwin in enumerate(isum):  
31
+            for iboot in range(nboot//isum[ii]):
32
+                cs = np.random.randint(0,nc-nwin)
33
+                Means[ii,iboot] = np.mean( N[cs:cs+nwin] )
34
+        Means = np.ma.masked_less(Means, -9995)
35
+
36
+    else:
37
+        Means = np.zeros((len(isum), nboot))
38
+        for ii, nwin in enumerate(isum):  
39
+            for iboot in range(nboot):
40
+                cs = np.random.randint(0,nc-nwin)
41
+                Means[ii,iboot] = np.mean( N[cs:cs+nwin] )
42
+
43
+    return Means, np.array(isum)
44
+
45
+def gateIntegrate(T2D, T2T, gpd, sigma, stackEfficiency=2.):
46
+    """ Gate integrate the signal to gpd, gates per decade
47
+        T2D = the time series to gate integrate, complex 
48
+        T2T = the abscissa values 
49
+        gpd = gates per decade 
50
+        sigma = estimate of standard deviation for theoretical gate noise 
51
+        stackEfficiency = exponential in theoretical gate noise, 2 represents ideal stacking
52
+    """
53
+    
54
+    # use artificial time gates so that early times are fully captured
55
+    T2T0 = T2T[0]
56
+    T2TD = T2T[0] - (T2T[1]-T2T[0])
57
+    T2T -= T2TD
58
+    
59
+    #####################################
60
+    # calculate total number of decades #
61
+    # windows edges are approximate until binning but will be adjusted to reflect data timing, this 
62
+    # primarily impacts bins with a few samples  
63
+    nd = np.log10(T2T[-1]/T2T[0])               
64
+    tdd = np.logspace( np.log10(T2T[0]), np.log10(T2T[-1]), (int)(gpd*nd)+1, base=10, endpoint=True) 
65
+    tdl = tdd[0:-1]                 # approximate window left edges
66
+    tdr = tdd[1::]                  # approximate window right edges
67
+    td = (tdl+tdr) / 2.             # approximate window centres
68
+
69
+
70
+    Vars = np.zeros( len(td) ) 
71
+    htd = np.zeros( len(td), dtype=complex )
72
+    isum = np.zeros( len(td), dtype=int )  
73
+
74
+    ii = 0
75
+    for itd in range(len(T2T)):
76
+        if ( T2T[itd] > tdr[ii] ):
77
+            ii += 1
78
+            # correct window edges to centre about data 
79
+            tdr[ii-1] = (T2T[itd-1]+T2T[itd])*.5 
80
+            tdl[ii  ] = (T2T[itd-1]+T2T[itd])*.5
81
+        isum[ii] += 1
82
+        htd[ii] += T2D[ itd ]
83
+        Vars[ii] += sigma**2
84
+        
85
+    td = (tdl+tdr) / 2.             # actual window centres
86
+    sigma2 = np.sqrt( Vars * ((1/(isum))**stackEfficiency) ) 
87
+
88
+    # Reset abscissa where isum == 1 
89
+    # when there is no windowing going on 
90
+    td[isum==1] = T2T[0:len(td)][isum==1]
91
+
92
+    tdd = np.append(tdl, tdr[-1])
93
+
94
+    htd /= isum # average
95
+    T2T += T2TD # not used  
96
+    return td+T2TD, htd, tdd+T2TD, sigma2, isum  # centre abscissa, data, window edges, error 
97
+
98
+PhiD = []
99
+def invert(Time, t, v, sig, lambdastar):
100
+    """ helper function that simply calls logBarrier, here to allow for drop in repacement  
101
+    """
102
+    #model = logBarrier(Time.Genv, 1e-2*v, Time.T2Bins, MAXITER=5000, sigma=1e-2*sig, alpha=1e6, smooth="Both") 
103
+    model = logBarrier(Time.Genv, 1e-2*v, Time.T2Bins, lambdastar, MAXITER=750, sigma=1e-2*sig, alpha=1e6, smooth="Smallest") 
104
+    PhiD.append(model[2])
105
+    return model
106
+
107
+def gateTest(vc, vgc, pperlin, boot, lamdastar):
108
+    """ Performs gate integration and adds random noise 
109
+        vc = clean data (dense)
110
+        vgc = clean data at gates 
111
+        boot = if "boot" then bootstrap the gate noise 
112
+        lambdastar = l-curve or discrepency principle
113
+        pperlin = percent perlin noise, noise floor is maintained at 2.00 PU 
114
+    """
115
+
116
+    t = np.arange(2e-4, .3601, 2e-4)
117
+    zeta = np.pi / 3.
118
+    
119
+    v = np.copy(vc) # important!  
120
+
121
+    # Scaling factors to keep noise floor constant with increasing levels of 
122
+    # Perlin noise. These were determined using populations of 5,000 and hold to 
123
+    # two significant digits (i.e, 2.00 PU) 
124
+    PF = {0.0:0,\
125
+          2.5:.450,\
126
+          5.0:.6125,\
127
+          7.5:.765,\
128
+          10.0:.87375,\
129
+          12.5:.9725,\
130
+          15.0:1.05,\
131
+          17.5:1.1275,\
132
+          20.0:1.20,\
133
+          22.5:1.265,\
134
+          25.0:1.325}
135
+ 
136
+    # random noise
137
+    np.random.seed() # necessary for thread pool, otherwise all threads can get same numbers 
138
+    sigma = 2.*(1.-1e-2*pperlin)
139
+    eps = np.random.normal(0, sigma, len(t)) + \
140
+       1j*np.random.normal(0, sigma, len(t)) 
141
+    eps += PF[pperlin] * perlin(len(t), L=.3601, sigma_f=.005, sigma_r=0.72) # 1 PU std
142
+    v += eps
143
+
144
+    # Noise based on residual  
145
+    sigmahat = np.std( v.imag )
146
+    
147
+    gt, gd, we, err, isum = gateIntegrate(v.real, t, 20, sigmahat)
148
+    ge = np.copy(err)
149
+    if boot=="boot":
150
+        Means, isum2 = bootstrapWindows(v.imag, 20000, isum[isum!=1], adapt=True)
151
+        # STD 
152
+        #err[isum!=1] = np.ma.std(Means, axis=1, ddof=1)[isum!=1]
153
+        # MAD, only for windows > 1 
154
+        c = stats.norm.ppf(3./4.)
155
+        err[isum!=1] = np.ma.median(np.ma.abs(Means), axis=1) / c 
156
+    if boot=="uniform":
157
+        err = sigmahat
158
+ 
159
+    return gt, gd, gd-vgc, err, v.real, t, isum
160
+
161
+if __name__ == "__main__":
162
+
163
+    import multiprocessing 
164
+    import itertools 
165
+
166
+    from GJIPlot import *
167
+    import numpy as np 
168
+    import matplotlib.pyplot as plt 
169
+    from matplotlib.ticker import FormatStrFormatter
170
+    from matplotlib import ticker
171
+    from collections import OrderedDict
172
+
173
+    if len(sys.argv)<4:
174
+        print ( "Python script for generating plots used in GJI publication")
175
+        print ( "useage:")
176
+        print ( "python gateIntegrate4.py   NoiseType   Sigma_i  Lambda*   " )
177
+        exit()
178
+
179
+    if sys.argv[1] not in ['0.0','2.5','5.0','7.5','10.0','12.5','15.0','17.5','20.0','22.5','25.0']: 
180
+        print ( "PercentPerlin: [0.0,2.5,5.0...25.0] ", "got", sys.argv[1])
181
+        exit(1)
182
+
183
+    if sys.argv[2] != "gauss" and sys.argv[2] != "boot" and sys.argv[2] != "uniform":
184
+        print ( "Sigma_i: gauss | boot | uniform")
185
+        exit(1)
186
+    
187
+    if sys.argv[3] != "lcurve" and sys.argv[3] != "discrepency": 
188
+        print ( "Lambda*: lcurve | discrepency ")
189
+        exit(1)
190
+
191
+    #offwhite = (.98,.98,.98)
192
+    offwhite = (1.,1.,1.) 
193
+    mDarkBrown = '#eb811b' # alert colour 
194
+    mDarkTeal = '#23373b'
195
+    mLightBrown= "#EB811B"
196
+    mLightGreen = "#14B03D"
197
+
198
+    # Time series plot
199
+    fig = plt.figure(figsize=(pc2in(18),pc2in(18)), facecolor=offwhite)
200
+    ax2  = fig.add_axes([.195, .175, .750, .75], facecolor=offwhite)  # time
201
+    
202
+    # Main plot 
203
+    fig2 = plt.figure(figsize=(pc2in(20),pc2in(2*.5*20)))
204
+    ax1  = fig2.add_axes([.175, .410*1.5, .6,   .225*1.5])
205
+    ax1c = fig2.add_axes([.800, .410*1.5, .025, .225*1.5])
206
+    ax3  = fig2.add_axes([.175, .100*1.5, .495, .225*1.5], facecolor='None')
207
+    ax3r  = fig2.add_axes([.175, .100*1.5, .495, .225*1.5], facecolor='None', rasterized=True, sharex=ax3, sharey=ax3)
208
+    ax3b = fig2.add_axes([.825, .100*1.5, .1,   .225*1.5])
209
+
210
+    SIG = []
211
+    ER = []
212
+    GD = []
213
+    GT = []
214
+    V = []
215
+    MOD = []
216
+    CONV = []
217
+    PHID = []
218
+    PHIM = []
219
+    LSTAR = []
220
+    ns = 10000 #10000  #10000 # number of realizations for PDF 
221
+    ni = 5000  #5000  #1000  # number of inversions to plot 
222
+    t = np.arange(2e-4, .3601, 2e-4) # CMR sampling
223
+ 
224
+    #CMAP = cmocean.cm.solar
225
+    CMAP = cmocean.cm.gray_r
226
+    #CMAP = cmocean.cm.haline
227
+    #CMAP = cmocean.cm.tempo
228
+
229
+    ##############################################
230
+    # set up model 
231
+    lowT2 = .001
232
+    hiT2 = 1.0 
233
+    nT2 = 30
234
+    spacing = "Log_10"
235
+    Time = pwcTime()
236
+    Time.setT2(lowT2, hiT2, nT2, spacing) 
237
+    Time.setSampling( np.arange(2e-4, .3601, 2e-4) )
238
+    Time.generateGenv()
239
+    tmod = np.zeros(nT2)
240
+    tmod [8] = .15  # distribution centres...to be smoothed
241
+    tmod [20] = .1
242
+    for i in range(2):
243
+        tmod = np.convolve(tmod, np.array([.0625,.125,.1875,.25,.1875,.125,.0625]), 'same') 
244
+
245
+    vc = 100. * np.dot(Time.Genv, tmod) + 0j # in PU
246
+    gt, gd, we, err, isum = gateIntegrate(vc, t, 20, 3)
247
+
248
+    ##############################################
249
+    # Set up inversion 
250
+    Time = pwcTime()
251
+    Time.setT2(lowT2, hiT2, nT2, spacing) 
252
+    Time.setSampling( gt ) 
253
+    Time.generateGenv()
254
+    vgc = 100.*np.dot(Time.Genv, tmod) + 0j # in PU
255
+
256
+    # make the Pool of workers
257
+    print("pool gate integrate")
258
+    with multiprocessing.Pool() as pool: 
259
+        results = pool.starmap(gateTest, zip(np.tile(vc, (ns, 1)), np.tile(vgc, (ns,1)), itertools.repeat(eval(sys.argv[1])), \
260
+                                                                                         itertools.repeat(sys.argv[2]), \
261
+                                                                                         itertools.repeat(sys.argv[3])))
262
+    print("done pool gate integrate")
263
+   
264
+    # parse out results 
265
+    for i in range(ns):
266
+        gt,gd,ge,err,v,vt,isum = results[i] 
267
+        V.append(v.real)
268
+        GT.append(gt.real)
269
+        GD.append(gd.real)
270
+        ER.append( ge.real / err.real )
271
+        SIG.append( err.real )
272
+
273
+    print("pool inversions")
274
+    with multiprocessing.Pool() as pool: 
275
+        invresults = pool.starmap(invert, zip(itertools.repeat(Time), GT[0:ni], GD[0:ni], SIG[0:ni], itertools.repeat(sys.argv[3]) )) 
276
+    #print("done pool inversions",results[:][0])
277
+
278
+    # Parse results 
279
+    for i in range(ns):
280
+        #print("Sym %", round(100.*i/(float)(1)/(float)(ns)))
281
+        # invert
282
+        if i < ni:
283
+            #mod, conv, phid = invert(Time, gt, gd.real, err)
284
+            if sys.argv[3] == "discrepency":
285
+                mod, conv, phid_final = invresults[i] 
286
+            else:
287
+                mod, conv, phid_final, phim, phid, lstar = invresults[i] 
288
+            MOD.append(mod)
289
+            CONV.append(conv)
290
+            PHID.append(phid)
291
+            PHIM.append(phim)
292
+            LSTAR.append(lstar)
293
+
294
+    PHIM = np.array(PHIM)
295
+    PHID = np.array(PHID)
296
+    ER = np.array(ER)
297
+    MOD = np.array(MOD)
298
+    GD = np.array(GD)
299
+
300
+    ####################
301
+    # Time series plot #       
302
+    ax2.plot( 1e3*vt, V[0], color=mDarkTeal, label="$V_N$", linewidth=1, zorder=-32) #, rasterized=True) 
303
+    ax2.errorbar( 1e3*gt, GD[0], yerr=SIG[0], fmt='.', markersize=6, color=mLightBrown, label="$V_G$")
304
+    ax2.set_ylim([-10,30])
305
+    leg1 = ax2.legend( labelspacing=0.2, scatterpoints=1, numpoints=1, frameon=True )   
306
+    fixLeg(leg1)
307
+    ax2.set_xscale("log", nonposx='clip')  
308
+    ax2.set_ylabel(r"$V_N$ (PU)")
309
+    ax2.get_xaxis().set_major_formatter(FormatStrFormatter('%1.0f'))
310
+    ax2.set_xlabel("time (ms)")
311
+    deSpine(ax2)
312
+    fig.savefig( sys.argv[1] + "-" + sys.argv[2] + "-" + sys.argv[3] + "-ts.pdf", dpi=400, facecolor=offwhite,edgecolor=offwhite)
313
+
314
+    # histogram of error statistic
315
+    bins = np.linspace( -3, 3, 40, endpoint=True )
316
+    HIST = []
317
+    for i in range(0,np.shape(ER)[1]):
318
+        hist, edges = np.histogram(ER[:,i], bins=bins, density=False)        
319
+        HIST.append(hist)
320
+    HIST =  np.array(HIST)/(float)(ns) # normalize 
321
+        
322
+    im = ax1.pcolor(1e3*we, edges, HIST.T, cmap=CMAP, vmin=0, vmax=.1, rasterized=True)
323
+    im.set_edgecolor('face')
324
+    cb = plt.colorbar(im, ax1c, label=r"probability density", format=FormatStrFormatter('%1.2f'))
325
+    cb.solids.set_rasterized(True) 
326
+    tick_locator = ticker.MaxNLocator(nbins=4)
327
+    cb.locator = tick_locator
328
+    cb.update_ticks()
329
+
330
+    ax1.set_xscale("log", nonposx='clip')
331
+    ax1.get_xaxis().set_major_formatter(FormatStrFormatter('%1.0f'))
332
+    ax1.set_xlabel("time (ms)")
333
+    ax1.set_ylabel(r"gate error $\left( \left( {V_G - V_T} \right) / {\tilde{\sigma_i}} \right)$")
334
+
335
+    LMT2 = [] 
336
+    THETA = []
337
+    MODERR = []
338
+       
339
+    # plot a random sample of ns instead?  
340
+    for i in range(ni):
341
+        # plot log mean and amplitude
342
+        model = MOD[i] 
343
+        theta = np.sum( model )
344
+        LogMeanT2 = np.exp(np.sum( model * np.log( Time.T2Bins ) ) / theta ) 
345
+        LMT2.append(LogMeanT2)
346
+        THETA.append( np.sum(model)  )
347
+        MODERR.append( np.linalg.norm(model-tmod) )
348
+                
349
+    CONV = np.array(CONV)
350
+    THETA = np.array(THETA)
351
+    MOD = np.array(MOD)
352
+    MODERR = np.array(MODERR)
353
+        
354
+    #############################
355
+    # plot all models, 1 colour #
356
+    ires = ax3r.plot( 1e3*np.tile(Time.T2Bins, (np.sum(np.array(CONV)) ,1)).T , 1e2*MOD.T, color=mDarkTeal, alpha=.01, lw=.5, label="$\mathbf{f}_I$", zorder=0, rasterized=True) 
357
+    lns2, = ax3r.plot(1e3*Time.T2Bins, 1e2*tmod, color=mLightBrown, linewidth=2, label="$\mathbf{f}_T$")
358
+
359
+    handles, labels = ax3r.get_legend_handles_labels()
360
+    by_label = OrderedDict(zip(labels, handles))
361
+    leg3 = ax3r.legend(by_label.values(), by_label.keys(), labelspacing=0.2, scatterpoints=1, numpoints=1, frameon=True , loc="upper right")            
362
+    for line in leg3.get_lines():
363
+        line.set_linewidth(1)
364
+    for lh in leg3.legendHandles: 
365
+        lh.set_alpha(1)
366
+    fixLeg(leg3)
367
+
368
+    ###########################
369
+    # Error histogram on side #
370
+    ax3b.hist( 1e2*MODERR, bins='auto', orientation="horizontal", color=mDarkTeal, stacked=True, density=True, range=(0,20)) 
371
+    ax3b.axhline(1e2*np.mean(MODERR), linewidth=1.25, color=mLightBrown) #, color=CMAP(0.7), zorder=1)
372
+    deSpine(ax3b)
373
+    ax3b.set_xscale("log", nonposx='clip')  
374
+    ax3b.set_ylabel(r"$\Vert \mathbf{f}_I -\mathbf{f}_T \Vert$") # %(m$^3$/m$^3$)") #, color="C0")   
375
+    ax3b.set_xlabel("log probability\ndensity") #, color="C0")   
376
+    ax3.set_xlim( (1e3*Time.T2Bins[0], 1e3*Time.T2Bins[-1]) ) 
377
+    ax3.set_ylim( (0,5) )  
378
+    ax3.set_xlabel("$T_2$ (ms)")   
379
+    ax3.set_ylabel("partial water content (PU)") #, color="C0")   
380
+    ax3.set_xscale("log", nonposx='clip')  
381
+    ax3.get_xaxis().set_major_formatter(FormatStrFormatter('%1.0f'))
382
+    plt.setp(ax3r.get_xticklabels(), visible=False)
383
+    plt.setp(ax3r.get_yticklabels(), visible=False)
384
+    deSpine(ax3)
385
+    deSpine(ax3r)
386
+    
387
+    np.save("pperlin" + str(round(1e1*eval(sys.argv[1]))) + "-" + sys.argv[2] + "-" + sys.argv[3] + "-err", MODERR) 
388
+
389
+    plt.savefig("pperlin" + str(round(1e1*eval(sys.argv[1]))) + "-" + sys.argv[2] + "-" + sys.argv[3] + ".pdf", dpi=600, facecolor=offwhite, edgecolor=offwhite)
390
+    
391
+    plt.show()
392
+

+ 12
- 6
akvo/tressel/invertTA.py View File

@@ -447,8 +447,8 @@ def main():
447 447
             ax1.set_ylim( ifaces[-1], ifaces[0] )
448 448
 
449 449
             ax2 = ax1.twiny()
450
-            ax2.plot( np.sum(DINV, axis=1), (ifaces[1:]+ifaces[0:-1])/2 ,  color='red' )
451
-            ax2.set_xlabel(u"total water (m$^3$/m$^3$)")
450
+            ax2.plot( np.sum(DINV, axis=1), (ifaces[1:]+ifaces[0:-1])/2 ,  color='mediumblue' )
451
+            ax2.set_xlabel(u"total water (m$^3$/m$^3$)", color='mediumblue')
452 452
             ax2.set_ylim( ifaces[-1], ifaces[0] )
453 453
             ax2.xaxis.set_major_locator( MaxNLocator(nbins = 3) )   
454 454
             ax2.get_xaxis().set_major_formatter(FormatStrFormatter('%0.2f'))
@@ -506,14 +506,17 @@ def main():
506 506
 
507 507
 
508 508
     ax2 = ax1.twiny()
509
-    ax2.plot( np.sum(INV, axis=1), (ifaces[1:]+ifaces[0:-1])/2 ,  color='red' )
510
-    ax2.set_xlabel(u"total water (m$^3$/m$^3$)")
509
+    ax2.plot( np.sum(INV, axis=1), (ifaces[1:]+ifaces[0:-1])/2 ,  color='mediumblue' )
510
+    ax2.set_xlabel(u"NMR total water (m$^3$/m$^3$)", color='mediumblue')
511 511
     ax2.set_ylim( ifaces[-1], ifaces[0] )
512 512
     ax2.xaxis.set_major_locator( MaxNLocator(nbins = 3) )   
513 513
     ax2.get_xaxis().set_major_formatter(FormatStrFormatter('%0.2f'))
514 514
     #ax2.axhline( y=ifaces[SNRidx], xmin=0, xmax=1, color='black', linestyle='dashed'  )
515 515
     if CalcDOI:
516 516
         ax2.axhline( y=DOI, xmin=0, xmax=1, color='black', linestyle='dashed'  )
517
+        
518
+    ax2.tick_params(axis='x', colors='mediumblue')
519
+    plt.setp(ax2.get_xticklabels(), color="mediumblue")
517 520
 
518 521
     plt.savefig("akvoInversion.pdf")
519 522
 
@@ -596,8 +599,8 @@ def main():
596 599
         #ax1.xaxis.set_label_position('top') 
597 600
 
598 601
         ax2 = ax1.twiny()
599
-        ax2.plot( np.sum(INVc, axis=1), (ifaces[1:]+ifaces[0:-1])/2 ,  color='red' )
600
-        ax2.set_xlabel(u"total water (m$^3$/m$^3$)")
602
+        ax2.plot( np.sum(INVc, axis=1), (ifaces[1:]+ifaces[0:-1])/2 ,  color='mediumblue' )
603
+        ax2.set_xlabel(u"NMR total water (m$^3$/m$^3$)", color='mediumblue')
601 604
         ax2.set_ylim( ifaces[-1], ifaces[0] )
602 605
         ax2.xaxis.set_major_locator( MaxNLocator(nbins = 3) )   
603 606
         ax2.get_xaxis().set_major_formatter(FormatStrFormatter('%0.2f'))
@@ -606,6 +609,9 @@ def main():
606 609
             ax2.axhline( y=DOI, xmin=0, xmax=1, color='black', linestyle='dashed'  )
607 610
         #ax2.xaxis.set_label_position('bottom') 
608 611
         #fig.suptitle("Non linear inversion")
612
+        ax2.tick_params(axis='x', colors='mediumblue')
613
+        plt.setp(ax2.get_xticklabels(), color="mediumblue")
614
+
609 615
         plt.savefig("akvoInversionNL.pdf")
610 616
 
611 617
 

+ 325
- 0
akvo/tressel/logbarrier-nnls.py View File

@@ -0,0 +1,325 @@
1
+from __future__ import division
2
+import numpy as np
3
+from scipy.sparse.linalg import iterative  as iter
4
+from scipy.sparse.linalg import spilu  as ilu
5
+import scipy.sparse.linalg as spla
6
+from scipy.sparse import eye as seye
7
+import pylab 
8
+import pprint 
9
+from scipy.optimize import nnls 
10
+
11
+import matplotlib.pyplot as plt
12
+
13
+from akvo.tressel.SlidesPlot import * 
14
+
15
+def PhiB(mux, minVal, x):
16
+    phib = mux * np.abs( np.sum(np.log( x-minVal)) )
17
+    return phib
18
+
19
+def curvaturefd(x, y, t):
20
+    x1 = np.gradient(x,t) 
21
+    x2 = np.gradient(x1,t) 
22
+    y1 = np.gradient(y,t) 
23
+    y2 = np.gradient(y1,t) 
24
+    return np.abs(x1*y2 - y1*x2) / np.power(x1**2 + y1**2, 3./2)
25
+
26
+def curvatureg(x, y):
27
+    from scipy.ndimage import gaussian_filter1d
28
+    #first and second derivative
29
+    x1 = gaussian_filter1d(x, sigma=1, order=1)#, mode='constant', cval=x[-1])
30
+    x2 = gaussian_filter1d(x1, sigma=1, order=1)#, mode='constant', cval=y[-1])
31
+    y1 = gaussian_filter1d(y, sigma=1, order=1)#, mode='constant', cval=x1[-1])
32
+    y2 = gaussian_filter1d(y1, sigma=1, order=1)#, mode='constant', cval=y1[-1])
33
+    return np.abs(x1*y2 - y1*x2) / np.power(x1**2 + y1**2, 3./2)
34
+
35
+def logBarrier(A, b, T2Bins, lambdastar, x_0=0, xr=0, alpha=10, mu1=10, mu2=10, smooth=False, MAXITER=70, fignum=1000, sigma=1, callback=None):
36
+    """Impliments a log barrier Tikhonov solution to a linear system of equations 
37
+        Ax = b  s.t.  x_min < x < x_max. A log-barrier term is used for the constraint
38
+    """
39
+    # TODO input
40
+    minVal = 0.0
41
+    #maxVal = 1e8
42
+
43
+    Wd =    (np.eye(len(b)) / (sigma))        # Wd = eye( sigma )
44
+    WdTWd = (np.eye(len(b)) / (sigma**2))     # Wd = eye( sigma )
45
+
46
+    ATWdTWdA = np.dot(A.conj().transpose(), np.dot( WdTWd, A ))     # TODO, implicit calculation instead?
47
+    N = np.shape(A)[1]                        # number of model
48
+    M = np.shape(A)[0]                        # number of data
49
+    SIGMA   = .25 # .25 # lower is more aggresive relaxation of log barrier  
50
+    EPSILON = 1e-25 #1e-35 
51
+    #SIGMA   = .05 # .25 # lower is more aggresive relaxation of log barrier  
52
+    #EPSILON = 1e-10 #1e-35 
53
+
54
+    # reference model
55
+    if np.size(xr) == 1:
56
+        xr =  np.zeros(N)     
57
+    
58
+    # initial guess
59
+    if np.size(x_0) == 1:
60
+        x = 1e-10 + np.zeros(N)     
61
+    else:
62
+        x = 1e-10 + x_0
63
+        
64
+    # Construct model constraint base   
65
+    Phim_base = np.zeros( [N , N] ) 
66
+    a1 = .05     # smallest too
67
+    
68
+    # calculate largest term            
69
+    D1 = 1./abs(T2Bins[1]-T2Bins[0])
70
+    D2 = 1./abs(T2Bins[2]-T2Bins[1])
71
+    #a2 = 1. #(1./(2.*D1+D2))    # smooth
72
+    
73
+    if smooth == "Both":
74
+        #print ("Both small and smooth model")
75
+        for ip in range(N):
76
+            D1 = 0.
77
+            D2 = 0.
78
+            if ip > 0:
79
+                #D1 = np.sqrt(1./abs(T2Bins[ip]-T2Bins[ip-1]))**.5
80
+                D1 = (1./abs(T2Bins[ip]-T2Bins[ip-1])) #**2
81
+            if ip < N-1:
82
+                #D2 = np.sqrt(1./abs(T2Bins[ip+1]-T2Bins[ip]))**.5
83
+                D2 = (1./abs(T2Bins[ip+1]-T2Bins[ip])) #**2
84
+            if ip > 0:
85
+                Phim_base[ip,ip-1] =   -(D1)      
86
+            if ip == 0:
87
+                Phim_base[ip,ip  ] = 2.*(D1+D2)  
88
+            elif ip == N-1:
89
+                Phim_base[ip,ip  ] = 2.*(D1+D2) 
90
+            else:
91
+                Phim_base[ip,ip  ] = 2.*(D1+D2)
92
+            if ip < N-1:
93
+                Phim_base[ip,ip+1] =   -(D2)  
94
+        Phim_base /= np.max(Phim_base)            # normalize 
95
+        Phim_base += a1*np.eye(N)
96
+
97
+    elif smooth == "Smooth":
98
+        #print ("Smooth model")
99
+        for ip in range(N):
100
+            if ip > 0:
101
+                Phim_base[ip,ip-1] = -1    # smooth in log space
102
+            if ip == 0:
103
+                Phim_base[ip,ip  ] = 2.05   # Encourage a little low model
104
+            elif ip == N-1:
105
+                Phim_base[ip,ip  ] = 2.5   # Penalize long decays
106
+            else:
107
+                Phim_base[ip,ip  ] = 2.1   # Smooth and small
108
+            if ip < N-1:
109
+                Phim_base[ip,ip+1] = -1    # smooth in log space
110
+
111
+    elif smooth == "Smallest":
112
+        for ip in range(N):
113
+            Phim_base[ip,ip  ] = 1.
114
+    else: 
115
+        print("non valid model constraint:", smooth)
116
+        exit()
117
+    
118
+    Phi_m =  alpha*Phim_base
119
+    WmTWm = Phim_base # np.dot(Phim_base, Phim_base.T)            
120
+    b_pre = np.dot(A, x)
121
+    phid = np.linalg.norm( np.dot(Wd, (b-b_pre)) )**2
122
+    phim = np.linalg.norm( np.dot(Phim_base, (x-xr)) )**2
123
+
124
+    mu2 = phim
125
+    phib = PhiB(mu1, 0, x) 
126
+    mu1 = 1e-4* ((phid + alpha*phim) / phib)
127
+
128
+    PHIM = []
129
+    PHID = []
130
+    MOD = []
131
+
132
+    ALPHA = []
133
+    ALPHA.append(alpha)
134
+    #ALPHA = np.linspace( alpha, 1, MAXITER  )
135
+    for i in range(MAXITER):
136
+        #alpha = ALPHA[i]
137
+
138
+        Phi_m =  alpha*Phim_base
139
+        
140
+        # reset mu1 at each iteration 
141
+        # Calvetti -> No ; Li -> Yes   
142
+        # without this, non monotonic convergence occurs...which is OK if you really trust your noise 
143
+        mu1 = 1e-4* ((phid + alpha*phim) / phib) 
144
+
145
+        WmTWm = Phim_base # np.dot(Phim_base, Phim_base.T)            
146
+        phid_old = phid
147
+        inner = 0
148
+
149
+        First = True # guarantee entry 
150
+
151
+        xp = np.copy(x) # prior step x 
152
+            
153
+        b2a = np.dot(A.conj().transpose(), np.dot(WdTWd, b-b_pre) ) - alpha*np.dot(WmTWm,(x-xr))
154
+        xg = nnls(ATWdTWdA + Phi_m, b2a)
155
+        x = xg[0]
156
+        
157
+        #b_pre = np.dot(A, x)
158
+        #phid = np.linalg.norm( np.dot(Wd, (b-b_pre)))**2
159
+        #phim = np.linalg.norm( np.dot(Phim_base, (x-xr)) )**2
160
+
161
+        while ( (phib / (phid+alpha*phim)) > EPSILON  or First==True ):
162
+        #while ( False ):
163
+
164
+            First = False
165
+            # Log barrier, keep each element above minVal
166
+            X1 = np.eye(N) * (x-minVal)**-1           
167
+            X2 = np.eye(N) * (x-minVal)**-2         
168
+            
169
+            # Log barrier, keep sum below maxVal TODO normalize by component. Don't want to push all down  
170
+            #Y1 = np.eye(N) * (maxVal - np.sum(x))**-1           
171
+            #Y2 = np.eye(N) * (maxVal - np.sum(x))**-2         
172
+            
173
+            AA = ATWdTWdA + mu1*X2 + Phi_m 
174
+            M = np.eye( N ) * (1./np.diag(ATWdTWdA + mu1*X2 + Phi_m))
175
+            ##M = seye( N ).dot(1./np.diag(ATWdTWdA + mu1*X2 + Phi_m))
176
+
177
+            #print("Incomplete LU",flush=True)
178
+            #M2 = ilu(AA) #, drop_tol=None, fill_factor=None, drop_rule=None, permc_spec=None, diag_pivot_thresh=None, relax=None, panel_size=None, options=None)     
179
+            #ilu(AA, drop_tol=None, fill_factor=None, drop_rule=None, permc_spec=None, 
180
+                #diag_pivot_thresh=None, relax=None, panel_size=None, options=None)[source]      
181
+            #M = spla.LinearOperator(np.shape(AA), M2.solve) 
182
+            #print("dun ILU", flush=True)
183
+ 
184
+            # Solve system (newton step) (Li)
185
+            b2 = np.dot(A.conj().transpose(), np.dot(WdTWd, b-b_pre) ) + 2.*mu1*np.diag(X1) - alpha*np.dot(WmTWm,(x-xr))
186
+            ztilde = iter.cg(AA, b2, M=M) 
187
+            h = (ztilde[0].real) 
188
+
189
+            # TODO move nnls outside of this loop...use as starting point maybe?
190
+            #print("nnls", flush=True)
191
+            #xg = nnls(AA, b2)
192
+            #print("nnls done", flush=True)
193
+            #print("calling cg", flush=True)
194
+            #print("out of cg", flush=True)
195
+            #print(xg[0], ztilde[0] , flush=True)
196
+            #print(np.linalg.norm(xg[0] - ztilde[0]), flush=True )
197
+
198
+            
199
+            # Solve system (direct solution) (Calvetti) 
200
+            #b2 = np.dot(A.conj().transpose(), np.dot(WdTWd, b)) + 2.*mu1*np.diag(X1) - alpha*np.dot(WmTWm,(x-xr))
201
+            #ztilde = iter.cg(AA, b2, M=M, x0=xg[0]) 
202
+            #h = (ztilde[0].real - x) 
203
+
204
+            # step size
205
+            d = np.min( (1, 0.95 * np.min(x/np.abs(h+1e-120))) )
206
+            
207
+            ##########################################################
208
+            # Update and fix any over/under stepping
209
+            x += d*h
210
+        
211
+            # Determine mu steps to take
212
+            s1 = mu1 * (np.dot(X2, ztilde[0].real) - 2.*np.diag(X1))
213
+            #s2 = mu2 * (np.dot(Y2, ztilde[0].real) - 2.*np.diag(Y1))
214
+
215
+            # determine mu for next step
216
+            mu1 = SIGMA/N * np.abs(np.dot(s1, x))
217
+            #mu2 = SIGMA/N * np.abs(np.dot(s2, x))
218
+            
219
+            b_pre = np.dot(A, x)
220
+            phid = np.linalg.norm( np.dot(Wd, (b-b_pre)))**2
221
+            phim = np.linalg.norm( np.dot(Phim_base, (x-xr)) )**2
222
+            phib = PhiB(mu1, minVal, x)
223
+            inner += 1
224
+ 
225
+        PHIM.append(phim)      
226
+        PHID.append(phid)      
227
+        MOD.append(np.copy(x))  
228
+
229
+        # determine alpha
230
+        scale = 1.5*(len(b)/phid)
231
+        #alpha *= np.sqrt(scale)
232
+        alpha *= min(scale, .95) # was .85...
233
+        #print("alpha", min(scale, 0.99))
234
+        #alpha *= .99 # was .85...
235
+        ALPHA.append(alpha)
236
+        #alpha = ALPHA[i+1]
237
+            
238
+        print("inversion progress", i, alpha, np.sqrt(phid/len(b)), phim, flush=True)       
239
+        
240
+
241
+#         if np.sqrt(phid/len(b)) < 0.97: 
242
+#             ibreak = -1
243
+#             print ("------------overshot--------------------", alpha, np.sqrt(phid/len(b)), ibreak)
244
+#             alpha *= 2. #0
245
+#             x -= d*h
246
+#             b_pre = np.dot(A, x)
247
+#             phid = np.linalg.norm( np.dot(Wd, (b-b_pre)))**2
248
+#             phim = np.linalg.norm( np.dot(Phim_base, (x-xr)) )#**2
249
+#             mu1 = ((phid + alpha*phim) / phib)
250
+        if lambdastar == "discrepency": 
251
+            if np.sqrt(phid/len(b)) < 1.00 or alpha < 1e-5: 
252
+                ibreak = 1
253
+                print ("optimal solution found", alpha, np.sqrt(phid/len(b)), ibreak)
254
+                break
255
+        # slow convergence, bail and use L-curve 
256
+        # TI- only use L-curve. Otherwise results for perlin noise are too spurious for paper.  
257
+        if lambdastar == "lcurve":
258
+            if i > 4: 
259
+                kappa = curvaturefd(np.log(np.array(PHIM)), np.log(np.array(PHID)), ALPHA[0:i+1])#ALPHA[0:-1])
260
+                #kappa = curvatureg(np.log(np.array(PHIM)), np.log(np.array(PHID)))
261
+                print("max kappa", np.argmax(kappa), "distance from", i-np.argmax(kappa)) 
262
+            if i > 4 and (i-np.argmax(kappa)) > 4: # ((np.sqrt(phid_old/len(b))-np.sqrt(phid/len(b))) < 1e-4) : 
263
+            #if np.sqrt(phid/len(b)) < 3.0 and ((np.sqrt(phid_old/len(b))-np.sqrt(phid/len(b))) < 1e-3): 
264
+                ibreak = 1
265
+                MOD = np.array(MOD)
266
+                print ("###########################") #slow convergence", alpha, "phid_old", np.sqrt(phid_old/len(b)), "phid", np.sqrt(phid/len(b)), ibreak)
267
+                print ("Using L-curve criteria") 
268
+                #kappa = curvaturefd(np.log(np.array(PHIM)), np.log(np.array(PHID)), ALPHA[0:-1])
269
+                #kappa2 = curvatureg(np.log(np.array(PHIM)), np.log(np.array(PHID)))
270
+                #kappa = curvature( np.array(PHIM), np.array(PHID))
271
+                x = MOD[ np.argmax(kappa) ]
272
+                b_pre = np.dot(A, x)
273
+                phid = np.linalg.norm( np.dot(Wd, (b-b_pre)))**2
274
+                phim = np.linalg.norm( np.dot(Phim_base, (x-xr)) )**2
275
+                mu1 = ((phid + alpha*phim) / phib) 
276
+                print ("L-curve selected", alpha, "phid_old", np.sqrt(phid_old/len(b)), "phid", np.sqrt(phid/len(b)), ibreak)
277
+                print ("###########################")
278
+                if np.sqrt(phid/len(b)) <= 1:
279
+                    ibreak=0
280
+
281
+                fig = plt.figure( figsize=(pc2in(20.0),pc2in(22.)) )
282
+                ax1 = fig.add_axes( [.2,.15,.6,.7] )
283
+                #plt.plot( (np.array(PHIM)),  np.log(np.array(PHID)/len(b)), '.-')
284
+                #plt.plot(  ((np.array(PHIM))[np.argmax(kappa)]) , np.log( (np.array(PHID)/len(b))[np.argmax(kappa)] ), '.', markersize=12)
285
+                #plt.axhline()
286
+                lns1 = plt.plot( np.log(np.array(PHIM)),  np.log(np.sqrt(np.array(PHID)/len(b))), '.-', label="L curve")
287
+                lns2 = plt.plot( np.log(np.array(PHIM))[np.argmax(kappa)], np.log(np.sqrt(np.array(PHID)/len(b))[np.argmax(kappa)]), '.', markersize=12, label="$\lambda^*$")
288
+                ax2 = plt.twinx()
289
+                lns3 = ax2.plot( np.log(np.array(PHIM)), kappa, color='orange', label="curvature" )
290
+
291
+                # Single legend 
292
+                lns = lns1+lns3
293
+                labs = [l.get_label() for l in lns]
294
+                ax2.legend(lns, labs, loc=0)
295
+
296
+                ax1.set_xlabel("$\phi_m$")
297
+                ax1.set_ylabel("$\phi_d$")
298
+                
299
+                ax2.set_ylabel("curvature")
300
+
301
+                plt.savefig('lcurve.pdf')
302
+                break
303
+
304
+    PHIM = np.array(PHIM)
305
+    PHID = np.array(PHID)
306
+
307
+    if (i == MAXITER-1 ):
308
+        ibreak = 2
309
+        print("Reached max iterations!!", alpha, np.sqrt(phid/len(b)), ibreak)
310
+        kappa = curvaturefd(np.log(np.array(PHIM)), np.log(np.array(PHID)), ALPHA[0:-1])
311
+        x = MOD[ np.argmax(kappa) ]
312
+        b_pre = np.dot(A, x)
313
+        phid = np.linalg.norm( np.dot(Wd, (b-b_pre)))**2
314
+        phim = np.linalg.norm( np.dot(Phim_base, (x-xr)) )**2
315
+        mu1 = ((phid + alpha*phim) / phib) 
316
+
317
+    if lambdastar == "lcurve":
318
+        return x, ibreak, np.sqrt(phid/len(b)), PHIM, PHID/len(b), np.argmax(kappa)
319
+    else:
320
+        return x, ibreak, np.sqrt(phid/len(b))
321
+
322
+
323
+
324
+if __name__ == "__main__":
325
+    print("Test")

+ 239
- 38
akvo/tressel/mrsurvey.py View File

@@ -6,8 +6,9 @@ import sys
6 6
 import scipy
7 7
 from scipy import stats
8 8
 import copy
9
-import struct
9
+import struct, glob
10 10
 from scipy.io.matlab import mio
11
+import pandas as pd
11 12
 from numpy import pi
12 13
 from math import floor
13 14
 import matplotlib as mpl
@@ -243,6 +244,7 @@ class GMRDataProcessor(SNMRDataProcessor):
243 244
         self.nTransVersion   = HEADER[8]           # Transmitter version
244 245
         self.nDAQVersion     = HEADER[9]           # DAQ software version 
245 246
         self.nInterleaves    = HEADER[10]          # num interleaves
247
+        self.Instrument      = "GMR" 
246 248
 
247 249
         self.gain()
248 250
         
@@ -315,7 +317,9 @@ class GMRDataProcessor(SNMRDataProcessor):
315 317
                 istack = 0
316 318
                 for sstack in self.DATADICT["stacks"]:
317 319
                     if self.pulseType == "FID" or pulse == "Pulse 2":
318
-                        if floor(self.nDAQVersion) < 2:
320
+                        if self.Instrument == "MIDI 2":
321
+                            mod = 1
322
+                        elif self.Instrument == "GMR" and floor(self.nDAQVersion) < 2:
319 323
                             mod = 1
320 324
                         else:
321 325
                             mod = (-1.)**(ipm%2) * (-1.)**(sstack%2)
@@ -950,27 +954,36 @@ class GMRDataProcessor(SNMRDataProcessor):
950 954
         self.doneTrigger.emit() 
951 955
 
952 956
 
953
-    def sumData(self, canvas, fred):
957
+    def sumData(self, canvas, plusminus, sumAll):
958
+        print("In sumData", plusminus, sumAll)
959
+        if plusminus == "sum":
960
+            splus = "+"
961
+        else:
962
+            splus = "-"
963
+
954 964
         chans = copy.deepcopy(self.DATADICT[self.DATADICT["PULSES"][0]]["chan"]) #= np.array( ( self.DATADICT[pulse]["chan"][0], ) )
955 965
         nchan = len(chans)
956 966
         # Sum permutations of two channel combos
957 967
         for ich in range(nchan-1):
958 968
             for ch in chans[ich+1:]:
959
-                chsum = chans[ich] + "+" + ch
969
+                chsum = chans[ich] + splus + ch
960 970
                 for pulse in self.DATADICT["PULSES"]:
961 971
                     self.DATADICT[pulse][chsum] = {} 
962 972
                     for ipm in range(self.DATADICT["nPulseMoments"]):
963 973
                         self.DATADICT[pulse][chsum][ipm] = {} 
964 974
                         for istack in self.DATADICT["stacks"]:
965
-                            self.DATADICT[pulse][chsum][ipm][istack] = self.DATADICT[pulse][chans[ich]][ipm][istack] - self.DATADICT[pulse][ch][ipm][istack] 
966
-                    if chsum == "1+2":
975
+                            if plusminus == "sum":
976
+                                self.DATADICT[pulse][chsum][ipm][istack] = self.DATADICT[pulse][chans[ich]][ipm][istack] + self.DATADICT[pulse][ch][ipm][istack] 
977
+                            else:
978
+                                self.DATADICT[pulse][chsum][ipm][istack] = self.DATADICT[pulse][chans[ich]][ipm][istack] - self.DATADICT[pulse][ch][ipm][istack] 
979
+                    #if chsum == "1+2":
967 980
                         #self.DATADICT[pulse]["rchan"].pop()
968 981
                         #self.DATADICT[pulse]["rchan"].pop()
969
-                        self.DATADICT[pulse]["chan"].append(chsum)
982
+                    self.DATADICT[pulse]["chan"].append(chsum)
970 983
 
971 984
         # Sum all channels 
972
-        sumall = False
973
-        if sumall:
985
+        #sumall = False
986
+        if sumAll:
974 987
             chsum = ""
975 988
             for ch in chans:
976 989
                 chsum += ch + "+" 
@@ -1735,7 +1748,9 @@ class GMRDataProcessor(SNMRDataProcessor):
1735 1748
     def enableDSP(self):
1736 1749
         self.enableDSPTrigger.emit() 
1737 1750
 
1738
-    def adaptiveFilter(self, M, flambda, truncate, mu, PCA, canvas):
1751
+    def adaptiveFilter(self, M, flambda, truncate, mu, PCA, plot, canvas):
1752
+
1753
+        #plot = False
1739 1754
 
1740 1755
         canvas.reAx2(shx=False, shy=False)
1741 1756
         # ax1 is top plot of filter taps 
@@ -1762,13 +1777,14 @@ class GMRDataProcessor(SNMRDataProcessor):
1762 1777
         for istack in self.DATADICT["stacks"]:
1763 1778
             for ipm in range(self.DATADICT["nPulseMoments"]):
1764 1779
                 for pulse in self.DATADICT["PULSES"]:
1765
-                    canvas.softClear()
1766
-                    mmax = 0
1767
-                    for ichan in self.DATADICT[pulse]["chan"]:
1768
-                        canvas.ax2.plot( self.DATADICT[pulse]["TIMES"], 1e9* self.DATADICT[pulse][ichan][ipm][istack], alpha=.5) 
1769
-                        mmax = max(mmax, np.max(1e9*self.DATADICT[pulse][ichan][ipm][istack])) 
1770
-                    canvas.ax2.set_ylim(-mmax, mmax) 
1771
-                    canvas.ax2.set_prop_cycle(None)
1780
+                    if plot:
1781
+                        canvas.softClear()
1782
+                        mmax = 0
1783
+                        for ichan in self.DATADICT[pulse]["chan"]:
1784
+                            canvas.ax2.plot( self.DATADICT[pulse]["TIMES"], 1e9* self.DATADICT[pulse][ichan][ipm][istack], alpha=.5) 
1785
+                            mmax = max(mmax, np.max(1e9*self.DATADICT[pulse][ichan][ipm][istack])) 
1786
+                        canvas.ax2.set_ylim(-mmax, mmax) 
1787
+                        canvas.ax2.set_prop_cycle(None)
1772 1788
                     for ichan in self.DATADICT[pulse]["chan"]:
1773 1789
                         #H = np.zeros(M)
1774 1790
                         RX = []
@@ -1810,42 +1826,46 @@ class GMRDataProcessor(SNMRDataProcessor):
1810 1826
                                                         RX,\
1811 1827
                                                         M, mu, PCA, flambda, H[pulse][ichan])
1812 1828
                                 iloop += 1
1813
-                            print("Recalled ", iloop, "times with norm=", np.linalg.norm(hm1-H[pulse][ichan]))
1829
+                            #print("Recalled ", iloop, "times with norm=", np.linalg.norm(hm1-H[pulse][ichan]))
1814 1830
                         else:
1815 1831
                             [e,H[pulse][ichan]] = Filt.adapt_filt_Ref( self.DATADICT[pulse][ichan][ipm][istack][::-1],\
1816 1832
                                                         RX,\
1817 1833
                                                         M, mu, PCA, flambda, H[pulse][ichan])
1818 1834
                         # replace
1819 1835
                         if truncate:
1820
-                            canvas.ax2.plot( self.DATADICT[pulse]["TIMES"][0:itrunc], 1e9* e[::-1][0:itrunc],\
1821
-                                label = pulse + " ipm=" + str(ipm) + " istack=" + str(istack) + " ichan="  + str(ichan))
1836
+                            if plot:
1837
+                                canvas.ax2.plot( self.DATADICT[pulse]["TIMES"][0:itrunc], 1e9* e[::-1][0:itrunc],\
1838
+                                    label = pulse + " ipm=" + str(ipm) + " istack=" + str(istack) + " ichan="  + str(ichan))
1822 1839
                             self.DATADICT[pulse][ichan][ipm][istack] = e[::-1][0:itrunc]
1823 1840
                         else:
1824
-                            canvas.ax2.plot( self.DATADICT[pulse]["TIMES"], 1e9* e[::-1],\
1825
-                                label = pulse + " ipm=" + str(ipm) + " istack=" + str(istack) + " ichan="  + str(ichan))
1841
+                            if plot:
1842
+                                canvas.ax2.plot( self.DATADICT[pulse]["TIMES"], 1e9* e[::-1],\
1843
+                                    label = pulse + " ipm=" + str(ipm) + " istack=" + str(istack) + " ichan="  + str(ichan))
1826 1844
                             self.DATADICT[pulse][ichan][ipm][istack] = e[::-1]
1827
-                           
1828
-     
1829
-                        #canvas.ax1.plot( H[pulse][ichan].reshape(-1, len(RX)) ) # , label="taps") 
1830
-                        canvas.ax1.plot( H[pulse][ichan][::-1].reshape(M, len(RX), order='F' ) ) #.reshape(-1, len(RX)) ) # , label="taps") 
1831 1845
 
1832
-                        canvas.ax2.legend(prop={'size':10}, loc='upper right')
1833
-                        #canvas.ax2.legend(prop={'size':6}, loc='upper right')
1846
+                                               
1847
+                        if plot:
1848
+                            #canvas.ax1.plot( H[pulse][ichan].reshape(-1, len(RX)) ) # , label="taps") 
1849
+                            canvas.ax1.plot( H[pulse][ichan][::-1].reshape(M, len(RX), order='F' ) ) #.reshape(-1, len(RX)) ) # , label="taps") 
1834 1850
 
1835
-                        mh = np.max(np.abs( H[pulse][ichan] ))
1836
-                        canvas.ax1.set_ylim( -mh, mh )
1851
+                            canvas.ax2.legend(prop={'size':10}, loc='upper right')
1852
+                            #canvas.ax2.legend(prop={'size':6}, loc='upper right')
1837 1853
 
1838
-                        canvas.ax2.set_xlabel(r"time (s)", fontsize=10)
1839
-                        canvas.ax2.set_ylabel(r"signal (nV)", fontsize=10)
1854
+                            mh = np.max(np.abs( H[pulse][ichan] ))
1855
+                            canvas.ax1.set_ylim( -mh, mh )
1840 1856
 
1841
-                        canvas.ax1.set_xlabel(r"filter tap index", fontsize=10)
1842
-                        canvas.ax1.set_ylabel(r"tap amplitude", fontsize=10)
1857
+                            canvas.ax2.set_xlabel(r"time (s)", fontsize=10)
1858
+                            canvas.ax2.set_ylabel(r"signal (nV)", fontsize=10)
1843 1859
 
1844
-                    canvas.fig.tight_layout()
1845
-                    deSpine(canvas.ax1)
1846
-                    deSpine(canvas.ax2)
1860
+                            canvas.ax1.set_xlabel(r"filter tap index", fontsize=10)
1861
+                            canvas.ax1.set_ylabel(r"tap amplitude", fontsize=10)
1847 1862
 
1848
-                    canvas.draw()
1863
+                    if plot:
1864
+                        canvas.fig.tight_layout()
1865
+                        deSpine(canvas.ax1)
1866
+                        deSpine(canvas.ax2)
1867
+
1868
+                        canvas.draw()
1849 1869
 
1850 1870
                     # truncate the reference channels too, in case you still need them for something. 
1851 1871
                     # Otherwise they are no longer aligned with the data 
@@ -2484,6 +2504,187 @@ class GMRDataProcessor(SNMRDataProcessor):
2484 2504
                 self.DATADICT["Pulse 1"]["CURRENT"][ipm][istack] = DATA[:,2][nps:nps+npul] * invCGain
2485 2505
             nds += ndr
2486 2506
             nps += ndr 
2507
+    
2508
+    def readMIDI2Header(self, directory):
2509
+        """ Reads the header of the FID_Q1_D0_R1.dat which should be in very MIDI directory
2510
+        """
2511
+        print("Searching ", directory, "for Q files")
2512
+        self.QFiles = []
2513
+        for file in glob.glob(directory+'/FID_Q*R1.dat'):
2514
+            self.QFiles.append(file)
2515
+
2516
+        fidname = self.QFiles[0] # "/FID_Q1_D0_R1.dat"
2517
+
2518
+        HEADER = {}
2519
+        with open(fidname, 'rb') as FID:
2520
+            #print(FID.name)
2521
+            headerLine = 0 
2522
+            for i in range(21):
2523
+                tags  = FID.readline().split(b']')
2524
+                tags[0] = tags[0].strip(b'[')
2525
+                tags[1] = tags[1].decode().strip(  )
2526
+                HEADER[ ''.join(map(chr, tags[0])) ] = tags[1]
2527
+        #print(HEADER)
2528
+
2529
+        pulseTypeDict = {
2530
+            0 : lambda: "FID",
2531
+            2 : lambda: "T1",
2532
+            3 : lambda: "SPINECHO",
2533
+            4 : lambda: "4PhaseT1"
2534
+        }
2535
+        
2536
+        pulseLengthDict = {
2537
+            1 : lambda x: np.ones(1) * x,
2538
+            2 : lambda x: np.ones(2) * x,
2539
+            3 : lambda x: np.array([x, 2.*x]),
2540
+            4 : lambda x: np.ones(2) * x
2541
+        }
2542
+
2543
+        if HEADER["P2"] == "0":
2544
+            self.pulseType       = "FID"
2545
+        else:
2546
+            self.pulseType       = "T1"
2547
+
2548
+        self.transFreq       = float(HEADER[ 'Excitation frequency /Hz' ])
2549
+        self.maxBusV         = '24 V'
2550
+        self.pulseLength     = [ float(HEADER['P1'])/self.transFreq, float(HEADER['P2'])/self.transFreq] 
2551
+        self.interpulseDelay = 1e-3*float(HEADER["Delay /ms"])       # for T2, Spin Echo
2552
+        self.repetitionDelay = float(HEADER['Pause /s'])             # delay between q pulses 
2553
+        self.nPulseMoments   = len(self.QFiles)                           # Number of pulse moments per stack
2554
+        self.TuneCapacitance = 0                                     # tuning capacitance in uF
2555
+        self.nTransVersion   = "MIDI 2"                              # Transmitter version
2556
+        self.nDAQVersion     = HEADER["Software Revision"]           # DAQ software version 
2557
+        self.nInterleaves    = 0                                     # num interleaves
2558
+        self.Instrument = "MIDI 2" 
2559
+        self.datadir = directory 
2560
+        self.MIDIGain = HEADER["Gains"].split()
2561
+        # default 
2562
+        self.samp                   = float(HEADER["Data Rate /Hz"])        # sampling frequency
2563
+        self.dt                     = 1./self.samp                          # sampling rate 
2564
+
2565
+
2566
+
2567
+    def loadMIDI2(self, directory, procStacks, chanin, rchanin, FIDProc, canvas, deadTime, plot):
2568
+        """Reads a MRS MIDI2 experiment. 
2569
+        """
2570
+        canvas.reAx3(True,False)
2571
+
2572
+        chan = []
2573
+        for ch in chanin:
2574
+            chan.append(str(ch)) 
2575
+        
2576
+        rchan = []
2577
+        for ch in rchanin:
2578
+            rchan.append(str(ch)) 
2579
+
2580
+        #print(self.QFiles)
2581
+        # Set up the same structure as GMR 
2582
+        PULSES = [FIDProc]
2583
+        PULSES = ["Pulse 1"]
2584
+
2585
+        self.DATADICT = {}
2586
+        self.DATADICT["nPulseMoments"] = self.nPulseMoments
2587
+        self.DATADICT["stacks"] = procStacks
2588
+        self.DATADICT["PULSES"] = PULSES
2589
+        for pulse in PULSES: 
2590
+            self.DATADICT[pulse] = {}
2591
+            self.DATADICT[pulse]["chan"] = chan        # TODO these should not be a subet of pulse! for GMR all 
2592
+            self.DATADICT[pulse]["rchan"] = rchan      #      data are consistent 
2593
+            self.DATADICT[pulse]["CURRENT"] = {} 
2594
+            for ichan in np.append(chan,rchan):
2595
+                self.DATADICT[pulse][ichan] = {}
2596
+                for ipm in range(self.nPulseMoments):
2597
+                    self.DATADICT[pulse][ichan][ipm] = {} 
2598
+                    self.DATADICT[pulse]["CURRENT"][ipm] = {} 
2599
+                    for istack in procStacks:
2600
+                        pass 
2601
+                        #print("pulse", pulse, "ichan",type(ichan), ichan, "ipm", type(ipm), ipm, "istack",type(istack), istack)
2602
+                        #
2603
+                        #self.DATADICT[pulse][ichan][ipm][istack] = np.zeros(3)
2604
+                        #self.DATADICT[pulse]["CURRENT"][ipm][istack] = np.zeros(3) 
2605
+
2606
+        iistack = 0
2607
+        idead = int( (self.pulseLength[0]+deadTime) / self.dt)
2608
+
2609
+        for iq in range(self.nPulseMoments):
2610
+
2611
+            fidbase = self.datadir #+ self.QFiles[iq][0:-5]
2612
+            
2613
+            for istack in procStacks:
2614
+                #fidname = fidbase + str(iq).zfill(2) + ".dat"
2615
+                fidname = fidbase + "/FID_Q" + str(iq) + "_D0_R" + str(istack) + ".dat"
2616
+ 
2617
+                with open(fidname, 'rb') as FID:
2618
+                    #print(FID.name)
2619
+                    headerLine = 0 
2620
+                    for i in range(100):
2621
+                        line = FID.readline().strip()
2622
+                        headerLine += 1
2623
+                        if line == b'[Begin Data]':
2624
+                            break 
2625
+
2626
+                    # use numpy for now, consider pandas for faster read? 
2627
+                    DATA = np.genfromtxt(FID, skip_header=0, skip_footer=1 )
2628
+                    #DATA = pd.read_csv(fidname, skiprows=headerLine, skipfooter=1, sep='\t', encoding='ascii')
2629
+           
2630
+                    for ichan in np.append(chan,rchan):
2631
+                 
2632
+                        if int(ichan) <= 3: 
2633
+                            self.DATADICT["Pulse 1"][ichan][iq][istack] = DATA[idead:,int(ichan)] / float(self.MIDIGain[int(ichan)-1])
2634
+
2635
+                        elif int(ichan) > 3:     
2636
+                            self.DATADICT["Pulse 1"][ichan][iq][istack] = DATA[idead:,int(ichan)+1] / float(self.MIDIGain[int(ichan)-1])
2637
+                   
2638
+                        # truncate after dead time  
2639
+
2640
+                        self.DATADICT["Pulse 1"]["TIMES"] = DATA[idead:,0]
2641
+
2642
+                        # truncate until dead time
2643
+                        ipulse = int(self.pulseLength[0] / self.dt)
2644
+                        self.DATADICT["Pulse 1"]["PULSE_TIMES"] = DATA[0:ipulse,0]
2645
+                        self.DATADICT["Pulse 1"]["CURRENT"][iq][istack] = DATA[0:ipulse,4]
2646
+
2647
+                        if plot: 
2648
+                            canvas.softClear()                           
2649
+
2650
+                            for ichan in chan:
2651
+                                canvas.ax1.plot(self.DATADICT["Pulse 1"]["PULSE_TIMES"], self.DATADICT["Pulse 1"]["CURRENT"][iq][istack] , color='black')
2652
+                                canvas.ax3.plot(self.DATADICT["Pulse 1"]["TIMES"],       self.DATADICT["Pulse 1"][ichan][iq][istack], label="Pulse 1 FID data ch. "+str(ichan)) #, color='blue')
2653
+
2654
+                            for ichan in rchan:
2655
+                                canvas.ax2.plot(self.DATADICT["Pulse 1"]["TIMES"], self.DATADICT["Pulse 1"][ichan][iq][istack], label="Pulse 1 FID ref ch. "+str(ichan)) #, color='blue')
2656
+
2657
+                            # reference axis
2658
+                            canvas.ax2.tick_params(axis='both', which='major', labelsize=10)
2659
+                            canvas.ax2.tick_params(axis='both', which='minor', labelsize=10)
2660
+                            #canvas.ax2.xaxis.set_ticklabels([])
2661
+                            plt.setp(canvas.ax2.get_xticklabels(), visible=False)
2662
+                            canvas.ax2.legend(prop={'size':10}, loc='upper right')
2663
+                            canvas.ax2.set_title("stack "+str(istack)+" pulse index " + str(iq), fontsize=10)
2664
+                            canvas.ax2.set_ylabel("RAW signal [V]", fontsize=10)
2665
+
2666
+                            canvas.ax1.set_ylabel("Current (A)", fontsize=10) 
2667
+                            canvas.ax1.ticklabel_format(style='sci', scilimits=(0,0), axis='y') 
2668
+                            canvas.ax1.set_xlabel("time (s)", fontsize=10)
2669
+
2670
+                            canvas.ax3.legend(prop={'size':10}, loc='upper right')
2671
+                            canvas.ax3.set_ylabel("RAW signal [V]", fontsize=10)
2672
+                    
2673
+                            canvas.fig.tight_layout()
2674
+                            canvas.draw()
2675
+
2676
+                percent = (int) (1e2*((float)(iistack) / (len(procStacks)*self.nPulseMoments)))
2677
+                self.progressTrigger.emit(percent) 
2678
+                iistack += 1
2679
+
2680
+#                percent = (int) (1e2*((float)((iistack*self.nPulseMoments+ipm+1))  / (len(procStacks)*self.nPulseMoments)))
2681
+#                self.progressTrigger.emit(percent) 
2682
+#                iistack += 1
2683
+
2684
+        self.enableDSP()    
2685
+        self.doneTrigger.emit()
2686
+
2687
+
2487 2688
 
2488 2689
     def loadGMRASCIIT1( self, rawfname, istack ):
2489 2690
         """Based on the geoMRI instrument manufactured by VistaClara. Imports 

+ 1
- 1
setup.py View File

@@ -21,7 +21,7 @@ with open("README.md", "r") as fh:
21 21
     long_description = fh.read()
22 22
 
23 23
 setup(name='Akvo',     
24
-      version='1.6.3', 
24
+      version='1.7.0', 
25 25
       python_requires='>3.7.0', # due to pyLemma 
26 26
       description='Surface nuclear magnetic resonance workbench',
27 27
       long_description=long_description,

Loading…
Cancel
Save