From 5a3430722e513e42cfff5a8cc43b4b1d89c83ee6 Mon Sep 17 00:00:00 2001 From: Malcolm Date: Mon, 13 Apr 2026 15:20:24 -0700 Subject: [PATCH 1/2] reformat & burstify signature nortek .mat files --- Signature/burstSIGmat.m | 160 +++++++++++++++++++++++++++++++++++++ Signature/reformatSIGmat.m | 28 ++++--- 2 files changed, 177 insertions(+), 11 deletions(-) create mode 100644 Signature/burstSIGmat.m diff --git a/Signature/burstSIGmat.m b/Signature/burstSIGmat.m new file mode 100644 index 0000000..25600af --- /dev/null +++ b/Signature/burstSIGmat.m @@ -0,0 +1,160 @@ +function burstSIGmat(missiondir, outdir, swiftname) +% BURSTSIGMAT Group reformatted Signature mat files into 12-minute bursts +% starting at the top of the hour (5 bursts/hr). Data is treated as +% sequential and bursts are allowed to span multiple reformat files. +% +% burstSIGmat(missiondir) +% burstSIGmat(missiondir, outdir) +% burstSIGmat(missiondir, outdir, swiftname) +% +% Output files: {swiftname}_SIG_{ddMMMyyyy}_{HH}_{BB}.mat +% where BB is the burst index in the hour (01: :00-:12 ... 05: :48-:60). +% +% M.LeClair April 2026 + +if nargin<2 || isempty(outdir), outdir = fullfile(missiondir,'bursts'); end +if ~exist(outdir,'dir'), mkdir(outdir); end + +files = dir(fullfile(missiondir,'*_reformat.mat')); +if isempty(files), disp('No *_reformat.mat files found.'); return; end + +% Natural sort by (runIdx, chunkIdx): SWIFTNN_RRRR.ad2cp.00000_CC_reformat.mat +keys = nan(length(files),2); +for i=1:length(files) + tok = regexp(files(i).name,'_(\d+)\.ad2cp\.\d+_(\d+)_reformat\.mat$','tokens','once'); + if ~isempty(tok) + keys(i,:) = [str2double(tok{1}) str2double(tok{2})]; + end +end +[~,ord] = sortrows(keys); +files = files(ord); + +if nargin<3 || isempty(swiftname) + tok = regexp(files(1).name,'^(SWIFT\d+)','tokens','once'); + swiftname = tok{1}; +end + +burstDur = 12/1440; % days +avgBuf = []; burstBuf = []; echoMeta = []; +curStart = []; + +for i = 1:length(files) + fprintf('Loading %s\n', files(i).name); + S = load(fullfile(files(i).folder, files(i).name)); + if isfield(S,'avg') && ~isempty(S.avg), avgBuf = catStruct(avgBuf, S.avg); end + if isfield(S,'burst') && ~isempty(S.burst), burstBuf = catStruct(burstBuf, S.burst); end + if isfield(S,'echo'), echoMeta = S.echo; end + + if isempty(curStart) + t0 = earliest(avgBuf, burstBuf); + if isempty(t0), continue; end + dv = datevec(t0); + dv(5) = floor(dv(5)/12)*12; dv(6) = 0; + curStart = datenum(dv); + end + + isLast = (i == length(files)); + + while true + burstEnd = curStart + burstDur; + tmax = latest(avgBuf, burstBuf); + if isempty(tmax), break; end + if ~isLast && tmax < burstEnd, break; end + if tmax < curStart + curStart = burstEnd; continue; % gap: skip empty burst windows + end + avgOut = sliceStruct(avgBuf, curStart, burstEnd); + burstOut = sliceStruct(burstBuf, curStart, burstEnd); + if ~isempty(avgOut) || ~isempty(burstOut) + saveBurst(outdir, swiftname, curStart, avgOut, burstOut, echoMeta); + end + avgBuf = dropBefore(avgBuf, burstEnd); + burstBuf = dropBefore(burstBuf, burstEnd); + curStart = burstEnd; + if isLast && isempty(latest(avgBuf, burstBuf)), break; end + end +end +end + +% ----- helpers ----- + +function A = catStruct(A, B) +if isempty(A), A = B; return; end +fn = fieldnames(B); +for k = 1:numel(fn) + f = fn{k}; + if strcmp(f,'CellSize') || strcmp(f,'Blanking') + A.(f) = B.(f); + else + if ~isfield(A,f), A.(f) = B.(f); else, A.(f) = cat(1, A.(f), B.(f)); end + end +end +end + +function t = earliest(a, b) +t = []; +if ~isempty(a) && ~isempty(a.time), t(end+1,1) = a.time(1); end +if ~isempty(b) && ~isempty(b.time), t(end+1,1) = b.time(1); end +if ~isempty(t), t = min(t); end +end + +function t = latest(a, b) +t = []; +if ~isempty(a) && ~isempty(a.time), t(end+1,1) = a.time(end); end +if ~isempty(b) && ~isempty(b.time), t(end+1,1) = b.time(end); end +if ~isempty(t), t = max(t); end +end + +function S = sliceStruct(S, tstart, tend) +if isempty(S), return; end +idx = S.time >= tstart & S.time < tend; +if ~any(idx), S = []; return; end +fn = fieldnames(S); +for k = 1:numel(fn) + f = fn{k}; + if strcmp(f,'CellSize') || strcmp(f,'Blanking'), continue; end + v = S.(f); + if size(v,1) ~= numel(idx), continue; end + switch ndims(v) + case 2, S.(f) = v(idx,:); + case 3, S.(f) = v(idx,:,:); + case 4, S.(f) = v(idx,:,:,:); + end +end +end + +function S = dropBefore(S, tend) +if isempty(S), return; end +keep = S.time >= tend; +if all(keep), return; end +if ~any(keep), S = []; return; end +fn = fieldnames(S); +for k = 1:numel(fn) + f = fn{k}; + if strcmp(f,'CellSize') || strcmp(f,'Blanking'), continue; end + v = S.(f); + if size(v,1) ~= numel(keep), continue; end + switch ndims(v) + case 2, S.(f) = v(keep,:); + case 3, S.(f) = v(keep,:,:); + case 4, S.(f) = v(keep,:,:,:); + end +end +end + +function saveBurst(outdir, swiftname, tstart, avg, burst, echo) +dv = datevec(tstart); +dstr = datestr(tstart,'ddmmmyyyy'); +hh = sprintf('%02d', dv(4)); +bn = sprintf('%02d', floor(dv(5)/12)+1); +fname = sprintf('%s_SIG_%s_%s_%s.mat', swiftname, dstr, hh, bn); +daydir = fullfile(outdir, datestr(tstart,'yyyymmdd')); +if ~exist(daydir,'dir'), mkdir(daydir); end +out = fullfile(daydir, fname); +fprintf(' -> %s\n', fname); +saveVars = struct(); +if ~isempty(avg), saveVars.avg = avg; end +if ~isempty(burst), saveVars.burst = burst; end +if ~isempty(echo), saveVars.echo = echo; end +save(out, '-struct', 'saveVars'); +end diff --git a/Signature/reformatSIGmat.m b/Signature/reformatSIGmat.m index 425ef55..7da2d89 100644 --- a/Signature/reformatSIGmat.m +++ b/Signature/reformatSIGmat.m @@ -2,6 +2,7 @@ function reformatSIGmat(missiondir) % Reformats a Sig1000 mat file generated by the Nortek Signature Viewer % software to match the convensions of the lab structures % K.Zeiden June 2024 +% M.LeClair April 2026 - parallel & fix bfiles = dir(fullfile(missiondir, '**', '*.mat')); @@ -13,6 +14,11 @@ function reformatSIGmat(missiondir) for iburst = 1:nburst + % Initialize variables + avg = []; + burst = []; + echo = []; + disp(['Reformatting ' bfiles(iburst).name ]) vars = whos('-file',[bfiles(iburst).folder slash bfiles(iburst).name]); @@ -22,7 +28,9 @@ function reformatSIGmat(missiondir) disp('Burst file already in Lab format. Skipping...') continue elseif any(strcmp({vars.name},'Data')) && any(strcmp({vars.name},'Config')) - load([bfiles(iburst).folder slash bfiles(iburst).name]) + S = load([bfiles(iburst).folder slash bfiles(iburst).name]); + Config = S.Config; + Data = S.Data; else disp('No Data or Config structure found. Skipping...') continue @@ -145,7 +153,7 @@ function reformatSIGmat(missiondir) ename = ['Echo1Bin1_' num2str(Config.EchoSounder_Frequency1) 'kHz']; - burst.time = ([ename '_Time']); + burst.time = Data.([ename '_Time']); burst.SoundSpeed = Data.([ename '_Soundspeed']); burst.Temperature = Data.([ename '_Temperature']); burst.Pressure = Data.([ename '_Pressure']); @@ -238,14 +246,14 @@ function reformatSIGmat(missiondir) burst.AHRS_GyroZ = Data.BurstHR_AHRSGyroZ; % Single Beam HR - if ~Config.bursthr_enable5 + if ~Config.bursthr_enable5 && ~Config.bursthr_enable burst.VelocityData = Data.BurstHR_VelBeam5; burst.AmplitudeData = Data.BurstHR_AmpBeam5; burst.CorrelationData = Data.BurstHR_CorBeam5; % 5-beam HR - elseif Config.bursthr_enable5 + elseif Config.bursthr_enable5 && Config.bursthr_enable nbeams_burst = Config.burst_nBeams; burst.VelocityData = NaN(npings_burst,nbins_burst,nbeams_burst,'single'); @@ -280,17 +288,15 @@ function reformatSIGmat(missiondir) end %% Save + outname = [bfiles(iburst).folder slash bfiles(iburst).name(1:end-4) '_reformat.mat']; if exist('burst','var') && exist('echo','var') - save([bfiles(iburst).folder slash bfiles(iburst).name(1:end-4) '_reformat.mat'],... - 'burst','avg','echo') + save(outname, 'burst','avg','echo') elseif exist('burst','var') - save([bfiles(iburst).folder slash bfiles(iburst).name(1:end-4) '_reformat.mat'],... - 'burst','avg') + save(outname, 'burst','avg') else - save([bfiles(iburst).folder slash bfiles(iburst).name(1:end-4) '_reformat.mat'],... - 'avg'); + save(outname, 'avg'); end - + %% Clear structures clearvars avg burst echo From cde9649cfd9a89feecebca94814736c24ded8b80 Mon Sep 17 00:00:00 2001 From: Malcolm Date: Thu, 16 Apr 2026 15:12:54 -0700 Subject: [PATCH 2/2] Fix working --- Signature/burstSIGmat.m | 47 ++++++++++++++++++++++++++------------ Signature/reformatSIGmat.m | 4 ++-- 2 files changed, 35 insertions(+), 16 deletions(-) diff --git a/Signature/burstSIGmat.m b/Signature/burstSIGmat.m index 25600af..8dcfb2c 100644 --- a/Signature/burstSIGmat.m +++ b/Signature/burstSIGmat.m @@ -1,40 +1,59 @@ -function burstSIGmat(missiondir, outdir, swiftname) -% BURSTSIGMAT Group reformatted Signature mat files into 12-minute bursts -% starting at the top of the hour (5 bursts/hr). Data is treated as -% sequential and bursts are allowed to span multiple reformat files. +function burstSIGmat(missiondir, outdir, swiftname, burstMin) +% BURSTSIGMAT Group reformatted Signature mat files into fixed-length bursts +% aligned within the hour. Data is treated as sequential and bursts are +% allowed to span multiple reformat files. % % burstSIGmat(missiondir) % burstSIGmat(missiondir, outdir) % burstSIGmat(missiondir, outdir, swiftname) +% burstSIGmat(missiondir, outdir, swiftname, burstMin) +% +% burstMin: burst length in minutes (default 12). Best if it divides 60 +% evenly so bursts tile the hour cleanly (e.g. 5, 6, 10, 12, 15, 20, 30, 60). % % Output files: {swiftname}_SIG_{ddMMMyyyy}_{HH}_{BB}.mat -% where BB is the burst index in the hour (01: :00-:12 ... 05: :48-:60). +% where BB is the burst index in the hour (01 = first burst of the hour). % % M.LeClair April 2026 if nargin<2 || isempty(outdir), outdir = fullfile(missiondir,'bursts'); end +if nargin<4 || isempty(burstMin), burstMin = 12; end if ~exist(outdir,'dir'), mkdir(outdir); end files = dir(fullfile(missiondir,'*_reformat.mat')); if isempty(files), disp('No *_reformat.mat files found.'); return; end -% Natural sort by (runIdx, chunkIdx): SWIFTNN_RRRR.ad2cp.00000_CC_reformat.mat +% Natural sort by (runIdx, chunkIdx). Handles both +% SWIFTNN_RRRR.ad2cp.NNNNN_CC_reformat.mat +% Data.RRR.NNNNN.ad2cp.NNNNN_CC_reformat.mat keys = nan(length(files),2); for i=1:length(files) - tok = regexp(files(i).name,'_(\d+)\.ad2cp\.\d+_(\d+)_reformat\.mat$','tokens','once'); + tok = regexp(files(i).name,'(\d+)\.ad2cp\.\d+_(\d+)_reformat\.mat$','tokens','once'); if ~isempty(tok) keys(i,:) = [str2double(tok{1}) str2double(tok{2})]; end end -[~,ord] = sortrows(keys); +if any(isnan(keys(:))) + warning('burstSIGmat:unparsedNames', ... + '%d file(s) did not match expected naming pattern; falling back to name sort.', ... + nnz(any(isnan(keys),2))); + [~,ord] = sort({files.name}); +else + [~,ord] = sortrows(keys); +end files = files(ord); if nargin<3 || isempty(swiftname) tok = regexp(files(1).name,'^(SWIFT\d+)','tokens','once'); - swiftname = tok{1}; + if ~isempty(tok) + swiftname = tok{1}; + else + [~,swiftname] = fileparts(missiondir); + if isempty(swiftname), swiftname = 'SIG'; end + end end -burstDur = 12/1440; % days +burstDur = burstMin/1440; % days avgBuf = []; burstBuf = []; echoMeta = []; curStart = []; @@ -49,7 +68,7 @@ function burstSIGmat(missiondir, outdir, swiftname) t0 = earliest(avgBuf, burstBuf); if isempty(t0), continue; end dv = datevec(t0); - dv(5) = floor(dv(5)/12)*12; dv(6) = 0; + dv(5) = floor(dv(5)/burstMin)*burstMin; dv(6) = 0; curStart = datenum(dv); end @@ -66,7 +85,7 @@ function burstSIGmat(missiondir, outdir, swiftname) avgOut = sliceStruct(avgBuf, curStart, burstEnd); burstOut = sliceStruct(burstBuf, curStart, burstEnd); if ~isempty(avgOut) || ~isempty(burstOut) - saveBurst(outdir, swiftname, curStart, avgOut, burstOut, echoMeta); + saveBurst(outdir, swiftname, curStart, avgOut, burstOut, echoMeta, burstMin); end avgBuf = dropBefore(avgBuf, burstEnd); burstBuf = dropBefore(burstBuf, burstEnd); @@ -142,11 +161,11 @@ function burstSIGmat(missiondir, outdir, swiftname) end end -function saveBurst(outdir, swiftname, tstart, avg, burst, echo) +function saveBurst(outdir, swiftname, tstart, avg, burst, echo, burstMin) dv = datevec(tstart); dstr = datestr(tstart,'ddmmmyyyy'); hh = sprintf('%02d', dv(4)); -bn = sprintf('%02d', floor(dv(5)/12)+1); +bn = sprintf('%02d', floor(dv(5)/burstMin)+1); fname = sprintf('%s_SIG_%s_%s_%s.mat', swiftname, dstr, hh, bn); daydir = fullfile(outdir, datestr(tstart,'yyyymmdd')); if ~exist(daydir,'dir'), mkdir(daydir); end diff --git a/Signature/reformatSIGmat.m b/Signature/reformatSIGmat.m index 7da2d89..95fd9f9 100644 --- a/Signature/reformatSIGmat.m +++ b/Signature/reformatSIGmat.m @@ -245,8 +245,8 @@ function reformatSIGmat(missiondir) burst.AHRS_GyroY = Data.BurstHR_AHRSGyroY; burst.AHRS_GyroZ = Data.BurstHR_AHRSGyroZ; - % Single Beam HR - if ~Config.bursthr_enable5 && ~Config.bursthr_enable + % Single Beam HR (vertical beam only) + if Config.bursthr_enable5 && ~Config.bursthr_enable burst.VelocityData = Data.BurstHR_VelBeam5; burst.AmplitudeData = Data.BurstHR_AmpBeam5;