Commit 8adef79b authored by Jay Dubb's avatar Jay Dubb
Browse files

v1.21.6 

-- Step 4 of changing the way Homer3 stores the group data in memory, how it saves processed data and loads processed and acquired data. Implemented distribute saving of derived data in separate files.
-- Add dataStorageScheme variable to DataTreeCass and make it a config option in AppSettings.cfg to be able to turn distributed saving of processed data on and off.
-- Remove testing of .snirf file with *_nirs user functions. Fundamentally incompatible. The *_nirs users functions should be tested only with .nirs files.
-- Reverse change to disable MainGUI when it is busy processing data, because there's a more optimal way to do this.
parent bcd07d9b

AppSettings.cfg

+3 −0
Original line number Diff line number Diff line
@@ -20,4 +20,7 @@ On 
% Check For Updates

on



% Data Storage Scheme

memory



% END

DataTree/AcquiredData/AcqDataClass.m

+23 −8
Original line number Diff line number Diff line
@@ -69,6 +69,21 @@ classdef AcqDataClass < matlab.mixin.Copyable 
    

    methods

        

        % -------------------------------------------------------

        function FreeMemory(obj, filename)

            if ~exist('filename','var')

                filename = '';

            end

            if isempty(filename)

                return;

            end

            

            % TBD: Tight now we only save processed output and free memory 

            % associated with it. Need to do the same for acquired data

            % obj.Initialize();  

        end

        

        

        % ---------------------------------------------------------

        function bbox = GetSdgBbox(obj)

            optpos = [obj.GetSrcPos(); obj.GetDetPos()];

DataTree/AcquiredData/Nirs/NirsClass.m

+12 −6
Original line number Diff line number Diff line
@@ -46,12 +46,7 @@ classdef NirsClass < AcqDataClass & FileLoadSaveClass 
            %

            

            % Initialize Nirs public properties

            obj.SD        = struct([]);

            obj.t         = [];

            obj.s         = [];

            obj.d         = [];

            obj.aux       = [];

            obj.CondNames = {};

            obj.Initialize();



            % Set base class properties not part of NIRS format 

            obj.filename  = '';
@@ -75,6 +70,17 @@ classdef NirsClass < AcqDataClass & FileLoadSaveClass 
        end

        

        

        % -------------------------------------------------------

        function Initialize(obj)

            obj.SD        = struct([]);

            obj.t         = [];

            obj.s         = [];

            obj.d         = [];

            obj.aux       = [];

            obj.CondNames = {};

        end

        

        

        % ---------------------------------------------------------

        function SortStims(obj)

            [~,idx] = sort(obj.CondNames);

DataTree/AcquiredData/Snirf/SnirfClass.m

+13 −7
Original line number Diff line number Diff line
@@ -65,12 +65,7 @@ classdef SnirfClass < AcqDataClass & FileLoadSaveClass 
            %

             

            % Initialize properties from SNIRF spec 

            obj.formatVersion = '1.0';

            obj.metaDataTags   = MetaDataTagsClass().empty();

            obj.data           = DataClass().empty();

            obj.stim           = StimClass().empty();

            obj.probe          = ProbeClass().empty();

            obj.aux            = AuxClass().empty();

            obj.Initialize()

            

            % Set class properties NOT part of the SNIRF format

            obj.fileformat = 'hdf5';
@@ -182,6 +177,17 @@ classdef SnirfClass < AcqDataClass & FileLoadSaveClass 
        

        

        

        % -------------------------------------------------------

        function Initialize(obj)

            obj.formatVersion = '1.0';

            obj.metaDataTags   = MetaDataTagsClass().empty();

            obj.data           = DataClass().empty();

            obj.stim           = StimClass().empty();

            obj.probe          = ProbeClass().empty();

            obj.aux            = AuxClass().empty();

        end

        

        

        % -------------------------------------------------------

        function err = Copy(obj, obj2)

            err=0;

DataTree/DataTreeClass.m

+74 −25
Original line number Diff line number Diff line
@@ -10,6 +10,7 @@ classdef DataTreeClass < handle 
        config

        logger

        warningflag

        dataStorageScheme

    end

    

    methods
@@ -25,6 +26,8 @@ classdef DataTreeClass < handle 
            obj.dirnameGroup        = '';

            obj.logger              = InitLogger(logger, 'DataTree');

                        

            cfg = ConfigFileClass();

            obj.dataStorageScheme = cfg.GetValue('Data Storage Scheme');

            

            %%%% Parse args
@@ -89,7 +92,7 @@ classdef DataTreeClass < handle 
            

            % Find index of another file format to try

            for ii = 1:length(supportedFormats)

                if ~isempty(findstr(dataInit.type, supportedFormats{ii,1}))

                if ~isempty(findstr(dataInit.type, supportedFormats{ii,1})) %#ok<FSTR>

                   k = ii;

                   break;

                end
@@ -177,7 +180,7 @@ classdef DataTreeClass < handle 
                    end

                    obj.files = dataInit.files;

                    

                    obj.LoadGroup(procStreamCfgFile);

                    obj.LoadGroups(procStreamCfgFile);

                    if length(obj.groups) < iGnew

                        if obj.FoundDataFilesInOtherFormat(dataInit) 

                            continue;
@@ -192,8 +195,37 @@ classdef DataTreeClass < handle 
        end

        

        

                

        % ---------------------------------------------------------------

        function SetDataStorageScheme(obj, iG)

            if length(obj.groups) < iG

                return

            end



            % Estimate memory requirement based on number of acquired files and their

            % average size

            % obj.logger.Write(sprintf('Memory required for data tree: %0.1f MB\n', obj.MemoryRequired() / 1e6));            

            if strcmp(obj.dataStorageScheme, 'disk')

                onoff = true;

            elseif strcmp(obj.dataStorageScheme, 'memory') || strcmpi(obj.dataStorageScheme, 'ram')

                onoff = false;

            else

                onoff = false;

            end

            obj.groups(iG).SaveMemorySpace(onoff);

            

            % If storage scheme is to store everything in memory, then

            % reload data set this time with the SavememorySpace variable 

            % turned off.

            if onoff == false

                obj.groups(iG).Load('reload');

                obj.groups(iG).SetConditions();

            end

        end 

        

          

        % ---------------------------------------------------------------

        function LoadGroup(obj, procStreamCfgFile)

        function LoadGroups(obj, procStreamCfgFile)

            if ~exist('procStreamCfgFile','var')

                procStreamCfgFile = '';

            end
@@ -210,34 +242,34 @@ classdef DataTreeClass < handle 
            obj.ErrorCheckLoadedFiles();



            tic;            

            for ii=1:length(obj.groups)

            for iG = 1:length(obj.groups)

                

                %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

                % Load derived or post-acquisition data from a file if it

                % exists

                %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

                obj.groups(ii).Load();            

                obj.groups(iG).Load();

            

                %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

                % Initialize procStream for all tree nodes

                %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

                obj.groups(ii).InitProcStream(procStreamCfgFile);

                obj.groups(iG).InitProcStream(procStreamCfgFile);

                

                %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

                % Generate the stimulus conditions for the group tree

                %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

                obj.groups(ii).SetConditions();                

            end

            obj.logger.Write(sprintf('Loaded processing stream results in %0.1f seconds\n', toc));

            

                obj.groups(iG).SetConditions();



                %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

            % Find the amount of memory the whole group tree requires

            % at the run level. If group runs take up more than half a

            % GB then do not save dc and dod time courses and recalculate

            % dc and dod for each new current element (currElem) on the

            % fly. This should be a menu option in future releases

                % Estimate amount of memory required and set the

                % data storage scheme

                %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

            % obj.logger.Write(sprintf('Memory required for data tree: %0.1f MB\n', obj.MemoryRequired() / 1e6));

                obj.SetDataStorageScheme(iG);

                

            end

            

            obj.logger.Write(sprintf('Loaded processing stream results in %0.1f seconds\n', toc));

            

        end
@@ -335,6 +367,15 @@ classdef DataTreeClass < handle 
        end        

        

        

        % ----------------------------------------------------------

        function LoadCurrElem(obj)

            if isempty(obj.groups)

                return;

            end

            obj.currElem.Load()

        end





        % ----------------------------------------------------------

        function SetCurrElem(obj, iGroup, iSubj, iRun)

            if isempty(obj.groups)
@@ -352,6 +393,14 @@ classdef DataTreeClass < handle 
                iRun  = 0;

            end



            if obj.currElem.IsSame(iGroup, iSubj, iRun)

                return;

            end

            

            % Free up memory of current element before reassigning it to

            % another node. 

            obj.currElem.FreeMemory();

            

            if iSubj==0 && iRun==0

                obj.currElem = obj.groups(iGroup);

            elseif iSubj>0 && iRun==0

CRA Git | Maintained and supported by SUSTech CRA and CCSE