Moriel

bento/data_builder/util/loadAnnotFileObserver.m

@@ -1,5 +1,6 @@
 function [annot, maxTime] = loadAnnotFileObserver(cleantext, fr)
 
+
 cleantext = cellfun(@(x) regexprep(x,{'\"','(',')'},''), cleantext, 'uniformoutput',false);
 
 headers = split(cleantext{1},';');
@@ -24,7 +25,7 @@
         flag = 1;
     elseif strcmpi(event,'state stop')
         flag = 2;
-    elseif strcmpi(event,'state point') % not sure what this means.
+    elseif strcmpi(event,'state point') % start==stop
         continue
     else % does this ever happen??
         flag = 0;

bento/data_builder/util/loadAnnotFileXls.m

@@ -0,0 +1,41 @@
+function [annot,maxTime,hotkeys,FR] = loadAnnotFileXls(fname,defaultFR, tmin,tmax)
+%
+% (C) Ann Kennedy, 2019
+% California Institute of Technology
+% Licensing: https://github.com/annkennedy/bento/blob/master/LICENSE.txt
+
+if(nargin<2)
+    defaultFR = 1000;
+elseif(nargin<3)
+    tmin = 1;
+    tmax = inf;
+elseif(isstr(tmin))
+    tmin = str2num(tmin);
+    tmax = str2num(tmax);
+end
+FR = nan; %framerate not usually specified in these kinds of files, return nan to let the gui decide.
+
+fid = fopen(fname);
+if(fid==-1)
+    keyboard
+end
+
+[~,sheetlist]   = xlsfinfo(fname);
+fid = sheetlist{1};  % let's assume there's just one sheet for now- go back and fix this later
+[~,~,M]      = xlsread(fname,fid);
+
+hotkeys=struct();
+
+if(any(strcmpi(M(1,:),'Event_type'))) % it's probably an Ethovision Observer log
+    [annot,maxTime]  = loadAnnotSheetObserver(M, defaultFR);  % need to add tmin/tmax support
+    FR = defaultFR;
+elseif(any(strcmpi(M(1,:), 'Sinuosity')))  % it's probably DeepSqueak
+    if defaultFR<1024
+        defaultFR = 1024;
+    end
+    [annot,maxTime]  = loadAnnotSheetDeepSqueak(M, defaultFR);  % need to add tmin/tmax support
+    FR = defaultFR;
+else
+    annot=[];maxTime=[];hotkeys=struct();
+end
+end

bento/data_builder/util/loadAnnotSheetDeepSqueak.m

@@ -0,0 +1,47 @@
+function [annot, maxTime] = loadAnnotSheetDeepSqueak(M, defaultFR)
+
+headers = M(1,:);
+label_index = find(strcmpi(headers,'Label'));
+accepted_index = find(strcmpi(headers,'Accepted'));
+time_start = find(strcmpi(headers,'Begin Time (s)')|strcmpi(headers,'BeginTime_s_'));
+time_stop = find(strcmpi(headers,'End Time (s)')|strcmpi(headers,'EndTime_s_'));
+cluster_index = find(strcmpi(headers,'cluster'));
+
+if length([label_index accepted_index time_start time_stop])~=4
+    disp('something went wrong- I couldn''t find fields Label, Accepted, Begin Time, and/or End Time.');
+    return
+end
+
+annot.Ch1 = [];
+tstop = 0;
+for i=2:length(M)
+    if (isstr(M{i,accepted_index}) && ~strcmpi(M{i,accepted_index},'TRUE')) || ~M{i,accepted_index}
+        continue;
+    end
+
+    if isempty(cluster_index)
+        beh = strrep(M{i, label_index},' ','_');
+    else
+        beh = strrep(M{i, label_index},' ','_');
+        beh = [beh '_' num2str(M{i,cluster_index})];
+    end
+    beh = regexprep(beh, '[^\w]', ''); % remove bad characters
+    if ~isfield(annot.Ch1,beh)
+        annot.Ch1.(beh) = [];
+    end
+    tstart = ceil(M{i, time_start}*defaultFR);
+    tstop = ceil(M{i, time_stop}*defaultFR);
+    annot.Ch1.(beh)(end+1,:) = [tstart tstop];
+end
+maxTime = tstop;
+
+
+
+
+
+
+
+
+
+
+

bento/data_builder/util/loadAnnotSheetObserver.m

@@ -0,0 +1,54 @@
+function [annot, maxTime] = loadAnnotSheetObserver(M, defaultFR)
+
+headers = M(1,:);
+behavior_name_index = find(strcmpi(headers,'behavior'));
+event_type_index = find(strcmpi(headers,'event_type'));
+timestamp_index = find(strcmpi(headers,'Time_Relative_sf'));
+
+if length([behavior_name_index event_type_index timestamp_index])~=3
+    disp('something went wrong- I couldn''t find fields Behavior, Event_Type, and/or Time_Relative_sf.');
+    return
+end
+
+annot.Ch1 = [];
+counts=[];
+for i=2:length(M)
+    event = M{i,event_type_index};
+    if isnan(event)
+        continue;
+    end
+    if strcmpi(event,'state start')
+        flag = 1;
+    elseif strcmpi(event,'state stop')
+        flag = 2;
+    elseif strcmpi(event,'state point') % start==stop
+        continue
+    else % does this ever happen??
+        flag = 0;
+        keyboard
+    end
+
+    beh = strrep(M{i, behavior_name_index},' ','_');
+    beh = regexprep(beh, '[^\w]', ''); % remove bad characters
+    time = ceil(M{i, timestamp_index}*defaultFR);
+    if ~isfield(annot.Ch1,beh)
+        annot.Ch1.(beh) = [];
+        counts.(beh) = 0;
+    end
+    if flag==1
+        counts.(beh) = counts.(beh)+1;
+    end
+    annot.Ch1.(beh)(counts.(beh),flag) = time;
+end
+maxTime = time;
+
+
+
+
+
+
+
+
+
+
+

bento/data_builder/util/loadAnnotSheetTxt.m

@@ -108,7 +108,7 @@
                 end
             end
             if((vals(2)+tmin)>=winStart && (vals(1)+tmax)<=winStop) %if this annotation is within the range we're loading
-                annot.(ch).(beh)(end+1,:) = min(max(vals(1:2)-winStart,1),winStop-winStart+1);
+                annot.(ch).(beh)(end+1,:) = min(max(vals(1:2)-winStart+1-tmin+1,0),winStop-winStart+1);
                 if(vals(2)-tmin+1 > tmax)
                     tmax = vals(2)-tmin+2;
                 end

bento/data_builder/util/loadAnyAnnot.m

@@ -11,29 +11,43 @@
 hotkeys = struct();
 
 switch ext
-    case '.xls' %old .annot format
-    fid     = filename;
-    if(loadRange)
-        [annotations,~] = loadAnnotSheet(filename,tmin,tmax);
-    else
-        [annotations,tmax] = loadAnnotSheet(filename);
-        tmin = 1;
-    end
-    FR = nan; %framerate not specified in file- trust user input to gui
-
     case '.annot' %new .annot format
-    fid = filename;
-    if(loadRange)
-        [annotations,~,~,FR] = loadAnnotSheetTxt(filename,tmin,tmax);
-    else
-        try
-        [annotations,tmin,tmax,FR] = loadAnnotSheetTxt(filename);
-        catch
-            keyboard
-            [annotations,tmax,tmin,fid,hotkeys] = deal([]);
-            errordlg(['Couldn''t load annotation file at ' filename]);
+        fid = filename;
+        if(loadRange)
+            [annotations,~,~,FR] = loadAnnotSheetTxt(filename,tmin,tmax);
+        else
+            try
+            [annotations,tmin,tmax,FR] = loadAnnotSheetTxt(filename);
+            catch
+                [annotations,tmax,tmin,fid,hotkeys] = deal([]);
+                errordlg(['Couldn''t load annotation file at ' filename]);
+            end
+        end
+    case {'.xls', '.xlsx'}
+        fid     = filename;
+        if(loadRange)
+            [annotations, ~, hotkeys, FR] = loadAnnotFileXls(filename, defaultFR, tmin, tmax);
+        else
+            [annotations,tmax,hotkeys,FR] = loadAnnotFileXls(filename, defaultFR);
+            tmin = 1;
+        end
+    case '.csv'  % Erin's grouped sequences
+        fid = filename;
+        M = dlmread(filename,',',1,0);
+        FR = 1024;  % 1024 is the framerate I use for deepsqueak
+        t_start = ceil(M(:,3)*FR);
+        t_stop = ceil(M(:,4)*FR);
+        drop = M(:,6)<2;
+        t_start(drop)=[];
+        t_stop(drop)=[];
+        tmin = 1;
+        tmax = t_stop(end);
+        annotations.USV_clusters.all = [t_start t_stop] + 0.382;  % whyyyy
+        for type = 0:5
+            inds = M(:,end)==type;
+            inds(drop)=[];
+            annotations.USV_clusters.(['type_' num2str(type)]) = [t_start(inds) t_stop(inds)] + round(0.382*FR); 
         end
-    end
 %     case 'csv' %temporary MUPET support
 %         fid = filename;
 %         M = dlmread(filename,',',1,0);
@@ -42,7 +56,7 @@
 %         tmin = 1;
 %         tmax = length(strtemp.audio.t);
 
-    case '.txt' %load data in the old format, OR ETHOVISION, prepare to convert to sheet format when saved
+    case '.txt'
     fid = filename;
     if(loadRange)
         [annotations,~,hotkeys,FR]    = loadAnnotFile(filename,defaultFR, tmin,tmax);
@@ -61,4 +75,5 @@
 
     otherwise
         [annotations,tmax,tmin,fid,hotkeys] = deal([]);
+        errordlg(['Unrecognized extension for annotation file ' filename]);
 end

bento/data_builder/util/unpackExperiment.m

@@ -337,7 +337,7 @@
                 [y,fs]          = audioread(fid);
                 disp('Generating spectrogram...');
                 win = hann(1024);
-                [~,f,t,psd]     = spectrogram(y,win,[],[],fs,'yaxis');
+                [~,f,t,psd]     = spectrogram(y,win,[],256,fs,'yaxis');
                 psd             = 10*log10(abs(double(psd)+eps));
                 disp('Saving spectrogram for future use...');
                 fid             = [pth strip(strip(strrep(data{i,match.Audio_file},ext,'_spectrogram.mat'),'left','.'),'left',filesep)];

bento/gui/readBehMovieFrame.m

@@ -36,7 +36,8 @@
                     time    = min(time, tMax);
                     frnum = round(time*reader{col,i}.reader.FrameRate);
                 end
-                reader{col,i}.reader.currentTime = min(time, reader{col,i}.reader.Duration - 1/reader{col,i}.reader.FrameRate);
+                reader{col,i}.reader.currentTime = min(time + 1/reader{col,i}.FrameRate, ...
+                                                    (reader{col,i}.NumFrames - 1)/reader{col,i}.FrameRate);
                 movies{col,i} = readFrame(reader{col,i}.reader);
 %                 movies{col,i} = read(reader{col,i},frnum);
 

bento/loading data/loadMovie.m

@@ -31,10 +31,15 @@
 %                                                 % inaccurate! trust the experimenter instead.
 %                 tMax            = reader{col,i}.numFrames/Fr;
             otherwise
-                reader{col,i}.reader       = VideoReader([data.io.movie.fid{col,i}]);
-                reader{col,i}.reader.CurrentTime = 0;
-                Fr              = reader{col,i}.reader.FrameRate;
-                tMax            = reader{col,i}.reader.Duration;
+                rtemp       = VideoReader([data.io.movie.fid{col,i}]);
+                rtemp.CurrentTime = 0;
+                Fr              = rtemp.FrameRate;
+                tMax            = rtemp.Duration;
+
+                reader{col,i}.reader = rtemp;
+                reader{col,i}.FrameRate = Fr;
+                reader{col,i}.Duration = tMax;
+                reader{col,i}.NumFrames = rtemp.NumFrames;
 
                 timestamps = getVideoTimestamps(data.io.movie.fid{col,i});
                 if timestamps

bento/loading data/unpackAnnotFromLoader.m

@@ -30,12 +30,7 @@
     if(subFrames)
         [atemp,tmax(j),tmin(j),FR,fid{j},~] = loadAnyAnnot([pth annoList{j}], defaultFR, startAnno,stopAnno);
     else
-        try
-            [atemp,tmax(j),tmin(j),FR,fid{j},~] = loadAnyAnnot([pth annoList{j}], defaultFR);
-        catch
-            disp(['Couldn''t load annotations at ' pth annoList{j}]);
-            keyboard
-        end
+        [atemp,tmax(j),tmin(j),FR,fid{j},~] = loadAnyAnnot([pth annoList{j}], defaultFR);
     end
 
     % generate a filename to save modified annotations

bento/loading data/unpackCaData.m

@@ -9,7 +9,7 @@
 [~,fname,ext] = fileparts(pth);
 disp(['Loading Ca file ' fname '...']);
 
-[rast,time,ROIs]=deal([]);
+[rast,time,ROIs,spikes]=deal([]);
 % add cases to this switch statement for other data types~?
 switch ext
     case '.csv'
@@ -21,6 +21,7 @@
 %             rast(i+1,:) = temp{6}(temp{1}==i);
 %             rast(i+1,temp{5}(temp{1}==i)==0)=nan;
 %         end
+
         temp = readtable(pth);
         if(size(temp,2)==5) % it's an Anderson lab miniscope file
             time = temp(:,1)';

bento/tracking/tracking unpackers/MARS.m

@@ -10,7 +10,7 @@
         v       = data.keypoints{fr};
         v       = permute(reshape(v,[2 2 7]),[2 1 3]);
     else
-        v       = squeeze(data.keypoints(fr,:,:,:));
+        v       = permute(data.keypoints(fr,:,:,:),[2,3,4,1]);
     end
 
     pts={};count=0;

bento/tracking/tracking unpackers/MARS_bbox_only.m

@@ -5,12 +5,14 @@
     if(iscell(data.bbox)) %support for both jsondecode and loadjson
         b       = data.bbox{fr};
     else
-        b       = squeeze(data.bbox(:,:,fr));
+        b       = permute(data.bbox(fr,:,:),[2,3,1]);
+    end
+
+    pts = {[]};
+    for i = 1:size(b,1)
+        bx     = [b(i,[1 3 3 1 1]); b(i,[2 2 4 4 2])];
+        pts{1} = [pts{1} [i bx(:)']];
     end
-    b1      = [b(1,[1 3 3 1 1]); b(1,[2 2 4 4 2])];
-    b2      = [b(2,[1 3 3 1 1]); b(2,[2 2 4 4 2])];
-
-    pts = {[1 b1(:)'] [2 b2(:)']};
 
     if(isfield(data,'crop_bounds') && ~isempty(data.crop_bounds)) % support for multi-arena MARS
         bounds = data.crop_bounds;