Unverified Commit 0e31b142 authored by jayd1860's avatar jayd1860 Committed by GitHub
Browse files

v1.78.0 -- Improve saving performance of SNIRF files by using low level HDF5 library. (#172) 

v1.78.0

* DataTree, v1.13.0
    -- Comprehensive solution to improving the save performance of SNIRF files by using low-level HDF5 calls and limiting the number of HDF5 open and create calls that are made for file, group and dataset.
parent 6089f97b

DataTree/AcquiredData/DataFiles/FileLoadSaveClass.m

+5 −0
Original line number Diff line number Diff line
@@ -63,6 +63,11 @@ classdef FileLoadSaveClass < matlab.mixin.Copyable 
                format = obj.fileformat;

            end

            

            p = fileparts(filename);

            if isempty(p)

                filename = ['./', filename];

            end

                       

            switch(lower(format))

                case obj.supportedFomats.matlab

                    if ismethod(obj, 'SaveMat')

DataTree/AcquiredData/DataFiles/Hdf5/hdf5write_safe.m

+142 −106
Original line number Diff line number Diff line 
function err = hdf5write_safe(fname, name, val, options)

    



function err = hdf5write_safe(fileobj, name, val, options)

err = -1;

if isempty(val)

    return;
@@ -16,13 +14,9 @@ function err = hdf5write_safe(fname, name, val, options) 
elseif any(strcmp(options, 'scalar'))

    force_scalar = true;

end

    % Identify type of val and use SNIRF v1.1-compliant write function



    if exist(fname,'file')

        fid = H5F.open(fname, 'H5F_ACC_RDWR', 'H5P_DEFAULT');

    else

        fid = H5F.create(fname, 'H5F_ACC_TRUNC', 'H5P_DEFAULT', 'H5P_DEFAULT');

    end

% Identify type of val and use SNIRF v1.1-compliant write function

fid = HDF5_GetFileDescriptor(fileobj);

if fid < 0

    err = -1;

    return;
@@ -36,87 +30,129 @@ function err = hdf5write_safe(fname, name, val, options) 
    return;

end



% Create dataset

if iscell(val) || isstring(val)

    if length(val) > 1 && ~force_scalar || force_array  % Returns true for single strings, believe it or not

            write_string_array(fid, fname, name, val);

        write_string_array(fid, name, val);

    else

            write_string(fid, fname, name, val);

        write_string(fid, name, val);

    end

        return

elseif ischar(val)

        write_string(fid, fname, name, val);

        return

    write_string(fid, name, val);

elseif isfloat(val)

    if length(val) > 1 && ~force_scalar || force_array

            write_numeric_array(fname, name, val);

        write_numeric_array(fid, name, val);

    else

            write_numeric(fid, fname, name, val);

        write_numeric(fid, name, val);

    end

        return

elseif isinteger(val)

    if length(val) > 1 && ~force_scalar || force_array

            write_numeric_array(fid, fname, name, val);  % As of now, no integer arrays exist

        write_numeric_array(fid, name, val);  % As of now, no integer arrays exist

    else

            write_integer(fid, fname, name, val);

        write_integer(fid, name, val);

    end

        return

else

        warning(['An unrecognized variable was saved to ', name, ' in ', fname])

    warning(['An unrecognized variable was saved to ', name])

end



    H5F.close(fid);



end



function err = write_string(fid, fname, name, val)

% -----------------------------------------------------------------

function err = write_string(fid, name, val)

sid = H5S.create('H5S_SCALAR');

tid = H5T.copy('H5T_C_S1');

H5T.set_size(tid, 'H5T_VARIABLE');

did = H5D.create(fid, name, tid, sid, 'H5P_DEFAULT');

H5D.write(did, tid, 'H5S_ALL', 'H5S_ALL', 'H5P_DEFAULT', {val});

err = 0;

end



function err = write_string_array(fid, fname, name, val)





% -----------------------------------------------------------------

function err = write_string_array(fid, name, val)

val = HDF5_Transpose(val);

sid = H5S.create_simple(1, numel(val), H5ML.get_constant_value('H5S_UNLIMITED'));

tid = H5T.copy('H5T_C_S1');

H5T.set_size(tid, 'H5T_VARIABLE');

pid = H5P.create('H5P_DATASET_CREATE');

H5P.set_chunk(pid, 2);

    did = H5D.create(fid, name, tid, sid, pid);

    H5D.write(did, tid, 'H5S_ALL', 'H5S_ALL', 'H5P_DEFAULT', val);

dsid = H5D.create(fid, name, tid, sid, pid);

H5D.write(dsid, tid, 'H5S_ALL', 'H5S_ALL', 'H5P_DEFAULT', val);

err = 0;

end



function err = write_numeric(fid, fname, name, val)

    fid = H5F.open(fname, 'H5F_ACC_RDWR', 'H5P_DEFAULT');





% -----------------------------------------------------------------

function err = write_numeric(fid, name, val)

tid = H5T.copy('H5T_NATIVE_DOUBLE');

sid = H5S.create('H5S_SCALAR');

    H5D.create(fid, name, tid, sid, 'H5P_DEFAULT');

    h5write(fname, name, val);

dsid = H5D.create(fid, name, tid, sid, 'H5P_DEFAULT');

H5D.write(dsid, tid, 'H5S_ALL', 'H5S_ALL', 'H5P_DEFAULT', val);

err = 0;

end



function err = write_numeric_array(fname, name, val)

    val = HDF5_Transpose(val);

    sizeval = size(val);

    if sizeval(1) == 1 || sizeval(2) == 1

        n = length(val);

    else

        n = sizeval;

    end

    h5create(fname, name, n, 'Datatype', 'double');

    h5write(fname, name, val);

    err = 0;

end



function err = write_integer(fid, fname, name, val)



% -----------------------------------------------------------------

function err = write_integer(fid, name, val)

warning off;  % Suppress the int truncation warning

    tid = H5T.copy('H5T_NATIVE_INT');

tid = H5T.copy('H5T_NATIVE_ULONG');

sid = H5S.create('H5S_SCALAR');

    H5D.create(fid, name, tid, sid, 'H5P_DEFAULT');

    h5write(fname, name, val);

dsid = H5D.create(fid, name, tid, sid, 'H5P_DEFAULT');

H5D.write(dsid, tid, 'H5S_ALL', 'H5S_ALL', 'H5P_DEFAULT', int32(val));

err = 0;

warning on;







% -----------------------------------------------------------------

function err = write_numeric_array(fid, name, data)

err = 0;

data = HDF5_Transpose(data);

sizedata = size(data);

if sizedata(1) == 1 || sizedata(2) == 1

    n = length(data);

else

    n = sizedata;

end



tid = -1;

sid = -1;

gid = -1;

dsid = -1;



maxdims = n;

try

    

    sid = H5S.create_simple(numel(n), fliplr(n), fliplr(maxdims));   

    gid = HDF5_CreateGroup(fid, fileparts(name));

    dsid = H5D.create(gid, name, 'H5T_NATIVE_DOUBLE', sid, 'H5P_DEFAULT');

    H5D.write(dsid, 'H5ML_DEFAULT', 'H5S_ALL', 'H5S_ALL', 'H5P_DEFAULT', data);

    

catch

    

    % Clean up; Close everything

    cleanUp(tid, sid, gid, dsid);

    err = -1;

    return;

    

end

cleanUp(tid, sid, gid, dsid);







% ------------------------------------------------------

function cleanUp(tid, sid, gid, dsid)

if ~isnumeric(tid)

    H5T.close(tid);

end

if ~isnumeric(sid)

    H5S.close(sid);

end

if ~isnumeric(gid)

    H5G.close(gid);

end

if ~isnumeric(dsid)

    H5D.close(dsid);

end

DataTree/AcquiredData/Nirs/NirsClass.m

+23 −12
Original line number Diff line number Diff line
@@ -1258,7 +1258,7 @@ classdef NirsClass < AcqDataClass & FileLoadSaveClass 
            % DetGrommetRot

            d2 = size(obj.SD.DetPos,1) - length(obj.SD.DetGrommetRot);

            if d2 > 0

                for ii = length(obj.SD.SrcGrommetRot)+1:length(obj.SD.SrcGrommetRot)+d2

                for ii = length(obj.SD.DetGrommetRot)+1:length(obj.SD.DetGrommetRot)+d2

                    if iscell(obj.SD.DetGrommetRot)

                        obj.SD.DetGrommetRot{ii} = 0;

                    else
@@ -1336,12 +1336,21 @@ classdef NirsClass < AcqDataClass & FileLoadSaveClass 
            obj.SD.DetPos3D = snirf.probe.detectorPos3D;

            obj.SD.MeasList = snirf.GetMeasList();

            obj.SD.SpatialUnit = snirf.GetLengthUnit();

            obj.SD.Landmarks.pos        = snirf.probe.landmarkPos2D;

            obj.SD.Landmarks2D.pos      = snirf.probe.landmarkPos2D;

            if length(snirf.probe.landmarkLabels) == size(snirf.probe.landmarkPos3D,1)

                obj.SD.Landmarks3D.labels   = snirf.probe.landmarkLabels;

            	obj.SD.Landmarks3D.pos      = snirf.probe.landmarkPos3D;

            obj.SD.Landmarks.labels     = snirf.probe.landmarkLabels;

            end

            if length(snirf.probe.landmarkLabels) == size(snirf.probe.landmarkPos2D,1)

            	obj.SD.Landmarks2D.labels   = snirf.probe.landmarkLabels;

            obj.SD.Landmarks3D.labels   = snirf.probe.landmarkLabels;

                obj.SD.Landmarks2D.pos      = snirf.probe.landmarkPos2D;

        	end

            if     ~isempty(obj.SD.Landmarks3D.labels)

                obj.SD.Landmarks.pos        = obj.SD.Landmarks3D.pos;

                obj.SD.Landmarks.labels     = obj.SD.Landmarks3D.labels;

            elseif ~isempty(obj.SD.Landmarks2D.labels)

                obj.SD.Landmarks.pos        = obj.SD.Landmarks2D.pos;

                obj.SD.Landmarks.labels     = obj.SD.Landmarks2D.labels;

            end                

        end
@@ -1391,8 +1400,10 @@ classdef NirsClass < AcqDataClass & FileLoadSaveClass 
        % ----------------------------------------------------------------------------------

        function ConvertSnirf(obj, snirf)

            obj.ConvertSnirfProbe(snirf);

            if ~isempty(snirf.data)

                obj.d = snirf.data(1).dataTimeSeries;

                obj.t = snirf.data(1).time;

            end

            obj.ConvertSnirfStim(snirf);

            obj.ConvertSnirfAux(snirf);

        end
@@ -1447,17 +1458,17 @@ classdef NirsClass < AcqDataClass & FileLoadSaveClass 
            end

            if isempty(obj.SD.SrcGrommetRot)

                for ii = 1:size(obj.SD.SrcPos,1)

                    obj.SD.SrcGrommetRot(ii) = 0;

                    obj.SD.SrcGrommetRot{ii} = 0;

                end

            end

            if isempty(obj.SD.DetGrommetRot)

                for ii = 1:size(obj.SD.DetPos,1)

                    obj.SD.DetGrommetRot(ii) = 0;

                    obj.SD.DetGrommetRot{ii} = 0;

                end

            end

            if isempty(obj.SD.DummyGrommetRot)

                for ii = 1:size(obj.SD.DummyPos,1)

                    obj.SD.DummyGrommetRot(ii) = 0;

                    obj.SD.DummyGrommetRot{ii} = 0;

                end

            end

            if isempty(obj.CondNames)

DataTree/AcquiredData/Snirf/AuxClass.m

+13 −9
Original line number Diff line number Diff line
@@ -116,7 +116,9 @@ classdef AuxClass < FileLoadSaveClass 


        

        % -------------------------------------------------------

        function SaveHdf5(obj, fileobj, location)

        function err = SaveHdf5(obj, fileobj, location)

            err = 0;

            

            % Arg 1

            if ~exist('fileobj', 'var') || isempty(fileobj)

                error('Unable to save file. No file name given.')
@@ -129,19 +131,21 @@ classdef AuxClass < FileLoadSaveClass 
                location = ['/',location];

            end

            

            if ~exist(fileobj, 'file')

                fid = H5F.create(fileobj, 'H5F_ACC_TRUNC', 'H5P_DEFAULT', 'H5P_DEFAULT');

                H5F.close(fid);

            % Convert file object to HDF5 file descriptor

            fid = HDF5_GetFileDescriptor(fileobj);

            if fid < 0

                err = -1;

                return;

            end

            

            if obj.debuglevel.Get() == obj.debuglevel.SimulateBadData()

                obj.SimulateBadData();

            end

            

            hdf5write_safe(fileobj, [location, '/name'], obj.name);

            hdf5write_safe(fileobj, [location, '/dataTimeSeries'], obj.dataTimeSeries, 'array');

            hdf5write_safe(fileobj, [location, '/time'], obj.time, 'array');

            hdf5write_safe(fileobj, [location, '/timeOffset'], obj.timeOffset, 'array');

            hdf5write_safe(fid, [location, '/name'], obj.name);

            hdf5write_safe(fid, [location, '/dataTimeSeries'], obj.dataTimeSeries, 'array');

            hdf5write_safe(fid, [location, '/time'], obj.time, 'array');

            hdf5write_safe(fid, [location, '/timeOffset'], obj.timeOffset, 'array');

        end

DataTree/AcquiredData/Snirf/DataClass.m

+22 −15
Original line number Diff line number Diff line
@@ -138,7 +138,6 @@ classdef DataClass < FileLoadSaveClass 
               return;

            end

            

            

            try

                % Open group

                [gid, fid] = HDF5_GroupOpen(fileobj, location);
@@ -183,6 +182,7 @@ classdef DataClass < FileLoadSaveClass 
        end

        

        

        

        % -------------------------------------------------------

        function err = LoadTime(obj, fileobj, location)

            err = 0;
@@ -205,12 +205,14 @@ classdef DataClass < FileLoadSaveClass 
            elseif isempty(fileobj)

                fileobj = obj.GetFilename();

            end

            if isempty(fileobj)

            

            % Convert file object to HDF5 file descriptor

            obj.fid = HDF5_GetFileDescriptor(fileobj);

            if obj.fid < 0

                err = -1;

                return;

            end

            

            

            try

                % Open group

                [gid, fid] = HDF5_GroupOpen(fileobj, location);
@@ -222,13 +224,13 @@ classdef DataClass < FileLoadSaveClass 
            catch

                err = -1;

            end

            

            err = ErrorCheck(obj, err, {'time'});

        end

        

        

        % -------------------------------------------------------

        function SaveHdf5(obj, fileobj, location)

        function err = SaveHdf5(obj, fileobj, location)

            err = 0;

            if ~exist('fileobj', 'var') || isempty(fileobj)

                error('Unable to save file. No file name given.')

            end
@@ -240,16 +242,17 @@ classdef DataClass < FileLoadSaveClass 
                location = ['/',location];

            end

            

            if ~exist(fileobj, 'file')

                fid = H5F.create(fileobj, 'H5F_ACC_TRUNC', 'H5P_DEFAULT', 'H5P_DEFAULT');

                H5F.close(fid);

            fid = HDF5_GetFileDescriptor(fileobj);

            if fid < 0

                err = -1;

                return;

            end

            

            hdf5write_safe(fileobj, [location, '/dataTimeSeries'], obj.dataTimeSeries, 'array');

            hdf5write_safe(fileobj, [location, '/time'], obj.time, 'array');

            hdf5write_safe(fid, [location, '/dataTimeSeries'], obj.dataTimeSeries, 'array');

            hdf5write_safe(fid, [location, '/time'], obj.time, 'array');

            

            for ii = 1:length(obj.measurementList)

                obj.measurementList(ii).SaveHdf5(fileobj, [location, '/measurementList', num2str(ii)]);

                obj.measurementList(ii).SaveHdf5(fid, [location, '/measurementList', num2str(ii)]);

            end

        end
@@ -1092,9 +1095,13 @@ classdef DataClass < FileLoadSaveClass 
            if ~all(size(obj.dataTimeSeries)==size(obj2.dataTimeSeries))

                return;

            end

            if ~all(obj.dataTimeSeries(:)==obj2.dataTimeSeries(:))

                obj.dataTimeSeries( isnan(obj.dataTimeSeries(:)) | isinf(obj.dataTimeSeries(:)) ) = 0;

                obj2.dataTimeSeries( isnan(obj2.dataTimeSeries(:)) | isinf(obj2.dataTimeSeries(:)) ) = 0;

                if ~all(obj.dataTimeSeries(:)==obj2.dataTimeSeries(:))

                    return;

                end

            end

            if ~all(obj.time(:)==obj2.time(:))

                return;

            end

CRA Git | Maintained and supported by SUSTech CRA and CCSE