Commit dbb39ff6 authored by jayd1860's avatar jayd1860
Browse files

v1.18.2 

-- Implement checking if there's enough disk space to save group results and if not, do NOT save, instead display error message.
-- Add checking for required disk space (as opposed to RAM) to GroupClass.MemoryRequired() method
-- Replace LogClass with improved Logger ported from a python project to log debug and diagnostic messages
parent 2df093d6

DataTree/AcquiredData/Snirf/SnirfClass.m

+3 −1
Original line number Diff line number Diff line
@@ -1106,7 +1106,9 @@ classdef SnirfClass < AcqDataClass & FileLoadSaveClass 
            for ii=1:length(obj.stim)

                nbytes = nbytes + obj.stim(ii).MemoryRequired();

            end

            if ~isempty(obj.probe)

                nbytes = nbytes + obj.probe.MemoryRequired();

            end

            for ii=1:length(obj.aux)

                nbytes = nbytes + obj.aux(ii).MemoryRequired();

            end

DataTree/DataTreeClass.m

+23 −10
Original line number Diff line number Diff line
@@ -8,17 +8,22 @@ classdef DataTreeClass < handle 
        currElem

        reg

        config

        logger

    end

    

    methods

        

        % ---------------------------------------------------------------

        function obj = DataTreeClass(groupDirs, fmt, procStreamCfgFile)

            global logger 

            

            obj.groups         = GroupClass().empty();

            obj.currElem      = TreeNodeClass().empty();

            obj.reg           = RegistriesClass().empty();

            obj.config        = ConfigFileClass().empty();

            obj.dirnameGroup  = '';

            obj.logger        = logger;

            

            

            %%%% Parse args
@@ -54,7 +59,7 @@ classdef DataTreeClass < handle 
            % Load user function registry

            obj.reg = RegistriesClass();

            if ~isempty(obj.reg.GetSavedRegistryPath())

                fprintf('Loaded saved registry %s\n', obj.reg.GetSavedRegistryPath());

                obj.logger.Write(sprintf('Loaded saved registry %s\n', obj.reg.GetSavedRegistryPath()));

            end

            

            % Initialize the current processing element within the group
@@ -71,20 +76,20 @@ classdef DataTreeClass < handle 




        % --------------------------------------------------------------

        function status = SelectOptionsWhenLoadFails(obj, dataInit)

        function status = SelectOptionsWhenLoadFails(obj, ~)

            status = -1;

            

            msg{1} = sprintf('Could not load any of the requested files in the group folder %s. ', obj.dirnameGroup);

            msg{2} = sprintf('Do you want to select another group folder?');

            q = MenuBox([msg{:}], {'YES','NO'});

            if q==2

                fprintf('Skipping group folder %s...\n', obj.dirnameGroup);

                obj.logger.Write(sprintf('Skipping group folder %s...\n', obj.dirnameGroup));

                obj.dirnameGroup = 0;

                return;

            end

            obj.dirnameGroup = uigetdir(pwd, 'Please select another group folder ...');

            if obj.dirnameGroup==0

                fprintf('Skipping group folder %s...\n', obj.dirnameGroup);

                obj.logger.Write(sprintf('Skipping group folder %s...\n', obj.dirnameGroup));

                return;

            end

            status = 0;
@@ -168,7 +173,7 @@ classdef DataTreeClass < handle 
                % dc and dod for each new current element (currElem) on the

                % fly. This should be a menu option in future releases

                %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

                fprintf('Memory required for acquisition data %0.1f MB\n', obj.groups(ii).MemoryRequired() / 1e6);

                % obj.logger.Write(sprintf('Memory required for acquisition data %0.1f MB\n', obj.groups(ii).MemoryRequired() / 1e6));

            end

        end
@@ -182,7 +187,7 @@ classdef DataTreeClass < handle 
            subjCurr = SubjClass().empty();

            runCurr = RunClass().empty();

            

            fprintf('\n');

            obj.logger.Write(sprintf('\n'));

            iG = 1;

            for iF=1:length(obj.files)

                % Extract group, subj, and run names from file struct
@@ -210,7 +215,7 @@ classdef DataTreeClass < handle 
                end

            end

            

            fprintf('\n');            

            obj.logger.Write(sprintf('\n'));            

        end
@@ -234,7 +239,12 @@ classdef DataTreeClass < handle 
                group.SetIndexID(jj);

                obj.groups(jj) = group;

                obj.groups(jj).SetPath(obj.dirnameGroup)

                fprintf('Added group %s to dataTree.\n', obj.groups(jj).GetName);

                

                % In case there's not enough disk space in the current

                % group folder, we have a separate path for saving group

                % results

                obj.groups(jj).SetPathOutput(obj.dirnameGroup)

                obj.logger.Write(sprintf('Added group %s to dataTree.\n', obj.groups(jj).GetName));

            end



            %v Add subj and run to group
@@ -304,8 +314,11 @@ classdef DataTreeClass < handle 




        % ----------------------------------------------------------

        function Save(obj)

            obj.groups(obj.currElem.iGroup).Save();

        function Save(obj, hwait)

            if ~exist('hwait','var')

                hwait = [];

            end

            obj.groups(obj.currElem.iGroup).Save(hwait);

        end

DataTree/GroupClass.m

+116 −38
Original line number Diff line number Diff line
@@ -4,11 +4,14 @@ classdef GroupClass < TreeNodeClass 
        version;

        versionStr;

        subjs;

        spaceRequired;

        spacesaver;

        logger

    end

    

    properties % (Access = private)

        path

        pathOutput

    end
@@ -19,15 +22,19 @@ classdef GroupClass < TreeNodeClass 
        

        % ----------------------------------------------------------------------------------

        function obj = GroupClass(varargin)

            global logger



            obj@TreeNodeClass(varargin);

            obj.logger = InitLogger(logger);

            obj.InitVersion();



            if nargin<3 || ~strcmp(varargin{3}, 'noprint')

                fprintf('Current GroupClass version %s\n', obj.GetVersionStr());

                obj.logger.Write(sprintf('Current GroupClass version %s\n', obj.GetVersionStr()));

            end

            

            obj.type    = 'group';

            obj.subjs   = SubjClass().empty;

            obj.spaceRequired = struct('memory',0, 'disk',0);

            obj.spacesaver = false;

            

            if nargin==0
@@ -53,7 +60,7 @@ classdef GroupClass < TreeNodeClass 
            if isempty(obj.name)

                % Derive obj name from the name of the root directory

                curr_dir = pwd;

                k = sort([findstr(curr_dir,'/') findstr(curr_dir,'\')]);

                k = sort([findstr(curr_dir,'/') findstr(curr_dir,'\')]); %#ok<*FSTR>

                obj.name = curr_dir(k(end)+1:end);

            end

        end
@@ -128,15 +135,12 @@ classdef GroupClass < TreeNodeClass 
                end

                v2 = str2cell(obj2.version,'.');

            elseif iscell(obj2.version)

                if ~isnumber([obj2.version{:}])

                    res = 1;

                end

                v2 = obj2.version;

            end

            v1 = obj.version;

            

            for ii=1:length(v1)

                v1{ii} = str2num(v1{ii});

                v1{ii} = str2num(v1{ii}); %#ok<*ST2NM>

            end

            for ii=1:length(v2)

                v2{ii} = str2num(v2{ii});
@@ -257,17 +261,33 @@ classdef GroupClass < TreeNodeClass 
        

        

        % ----------------------------------------------------------------------------------

        function nbytes = MemoryRequired(obj)

            nbytes = 0;

        function nbytes = MemoryRequired(obj, option)

            if ~exist('option','var')

                option = 'memory';

            end

            if strcmp(option, 'memory')

                nbytes = obj.spaceRequired.memory;

            else

                nbytes = obj.spaceRequired.disk;

            end

            if nbytes > 0

                return

            end

            if isempty(obj)

                return;

            end

            nbytes = obj.procStream.MemoryRequired();

            for ii=1:length(obj.subjs)

                nbytes = nbytes + obj.subjs(ii).MemoryRequired();

                nbytes = nbytes + obj.subjs(ii).MemoryRequired(option);

            end

            if nbytes > 5e8

                obj.spacesaver = true;

            end

            if strcmp(option, 'memory')

                obj.spaceRequired.memory = nbytes; 

            else

                obj.spaceRequired.disk = nbytes; 

            end

        end
@@ -277,6 +297,12 @@ classdef GroupClass < TreeNodeClass 
        end

        

        

        % ----------------------------------------------------------------------------------

        function SetPathOutput(obj, dirname)

            obj.pathOutput = dirname;

        end

        

        

        % ----------------------------------------------------------------------------------

        function Add(obj, subj, run)                        

            % Add subject to this group
@@ -291,7 +317,7 @@ classdef GroupClass < TreeNodeClass 
                jj = length(obj.subjs)+1;

                subj.SetIndexID(obj.iGroup, jj);

                obj.subjs(jj) = subj;

                fprintf('   Added subject %s to group %s.\n', obj.subjs(jj).GetName, obj.GetName);

                obj.logger.Write(sprintf('   Added subject %s to group %s.\n', obj.subjs(jj).GetName, obj.GetName));

            end

            

            % Add run to subj
@@ -304,7 +330,7 @@ classdef GroupClass < TreeNodeClass 
        function list = DepthFirstTraversalList(obj)

            list{1} = obj;

            for ii=1:length(obj.subjs)

                list = [list; obj.subjs(ii).DepthFirstTraversalList()];

                list = [list; obj.subjs(ii).DepthFirstTraversalList()]; %#ok<AGROW>

            end

        end
@@ -360,7 +386,7 @@ classdef GroupClass < TreeNodeClass 
                    procStreamRun.SaveConfigFile(fname, 'run');

                end

                

                fprintf('Attempting to load proc stream from %s\n', fname);

                obj.logger.Write(sprintf('Attempting to load proc stream from %s\n', fname));

                

                % Load file to the first empty procStream in the dataTree at each processing level

                g.LoadProcStreamConfigFile(fname);
@@ -374,18 +400,18 @@ classdef GroupClass < TreeNodeClass 
                % did not have valid proc stream input. If that's the case we

                % load a default proc stream input

                if g.procStream.IsEmpty() || s.procStream.IsEmpty() || r.procStream.IsEmpty()

                    fprintf('Failed to load all function calls in proc stream config file. Loading default proc stream...\n');

                    obj.logger.Write(sprintf('Failed to load all function calls in proc stream config file. Loading default proc stream...\n'));

                    g.CopyFcalls(procStreamGroup, 'group');

                    g.CopyFcalls(procStreamSubj, 'subj');

                    g.CopyFcalls(procStreamRun, 'run');

                    

                    % If user asked default config file to be generated ...

                    if autoGenDefaultFile

                        fprintf('Generating default proc stream config file %s\n', fname);

                        obj.logger.Write(sprintf('Generating default proc stream config file %s\n', fname));

                        

                        % Move exiting default config to same name with .bak extension

                        if ~exist([fname, '.bak'], 'file')

                            fprintf('Moving existing %s to %s.bak\n', fname, fname);

                            obj.logger.Write(sprintf('Moving existing %s to %s.bak\n', fname, fname));

                            movefile(fname, [fname, '.bak']);

                        end

                        procStreamGroup.SaveConfigFile(fname, 'group');
@@ -393,7 +419,7 @@ classdef GroupClass < TreeNodeClass 
                        procStreamRun.SaveConfigFile(fname, 'run');

                    end

                else

                    fprintf('Loading proc stream from %s\n', fname);

                    obj.logger.Write(sprintf('Loading proc stream from %s\n', fname));

                    procStreamGroup.Copy(g.procStream);

                    procStreamSubj.Copy(s.procStream);

                    procStreamRun.Copy(r.procStream);
@@ -418,7 +444,7 @@ classdef GroupClass < TreeNodeClass 
                obj.procStream.output.Flush();

            end

            if obj.DEBUG

                fprintf('Calculating processing stream for group %d\n', obj.iGroup)

                obj.logger.Write(sprintf('Calculating processing stream for group %d\n', obj.iGroup))

            end

            

            % Calculate all subjs in this session
@@ -468,8 +494,8 @@ classdef GroupClass < TreeNodeClass 
            obj.procStream.Calc();



            if obj.DEBUG

                fprintf('Completed processing stream for group %d\n', obj.iGroup);

                fprintf('\n');

                obj.logger.Write(sprintf('Completed processing stream for group %d\n', obj.iGroup));

                obj.logger.Write(sprintf('\n'));

            end

            

            % Update call application GUI using it's generic Update function
@@ -477,6 +503,10 @@ classdef GroupClass < TreeNodeClass 
                obj.updateParentGui('DataTreeClass', [obj.iGroup, obj.iSubj, obj.iRun]);

            end

            

            % Reset space required to zero so it will be recalculated in

            % the Save() method 

            obj.spaceRequired.memory = 0;

            obj.spaceRequired.disk = 0;

        end
@@ -497,7 +527,7 @@ classdef GroupClass < TreeNodeClass 
            if ~exist('indent', 'var')

                indent = 0;

            end

            fprintf('%sGroup 1:\n', blanks(indent));

            obj.logger.Write(sprintf('%sGroup 1:\n', blanks(indent)));

            obj.procStream.Print(indent+4);

            obj.procStream.output.Print(indent+4);

            for ii=1:length(obj.subjs)
@@ -545,6 +575,9 @@ classdef GroupClass < TreeNodeClass 
            if isempty(obj)

                return;

            end

            obj.spaceRequired.memory = 0;

            obj.spaceRequired.disk = 0;



            group = [];

            if exist([obj.path, 'groupResults.mat'],'file')

                g = load([obj.path, 'groupResults.mat']);
@@ -553,7 +586,7 @@ classdef GroupClass < TreeNodeClass 
                if isproperty(g, 'group') && isa(g.group, 'GroupClass')

                    if isproperty(g.group, 'version')

                        if ismethod(g.group, 'GetVersion')

                            fprintf('Saved group data, version %s exists\n', g.group.GetVersionStr());

                            obj.logger.Write(sprintf('Saved group data, version %s exists\n', g.group.GetVersionStr()));

                            group = g.group;

                        end

                    end
@@ -570,31 +603,76 @@ classdef GroupClass < TreeNodeClass 
                obj.Copy(group, 'conditional');

                close(hwait);

            else

                group = obj;

                group = obj; %#ok<NASGU>

                if exist([obj.path, 'groupResults.mat'],'file')

                    fprintf('Warning: This folder contains old version of groupResults.mat. Will move it to groupResults_old.mat\n');

                    obj.logger.Write(sprintf('Warning: This folder contains old version of groupResults.mat. Will move it to groupResults_old.mat\n'));

                    movefile([obj.path, 'groupResults.mat'], './groupResults_old.mat')

                end

                save([obj.path, 'groupResults.mat'],'group' );

                obj.Save();

            end

        end

        

        

        % ----------------------------------------------------------------------------------

        function [diskspaceToSpare, diskspacePercentRemaining] = CheckAvailableDiskSpace(obj, hwait)

            if ishandle(hwait)

                obj.logger.Write(sprintf('Calculating disk space required to save processing results ...\n'), obj.logger.ProgressBar(), hwait);

            end

            diskspaceToSpare = (getFreeDiskSpace() - obj.MemoryRequired('disk'));   % Disk space to spare in megabytes

            diskspacePercentRemaining = 100 * diskspaceToSpare/obj.MemoryRequired('disk');

            msg = {};

            obj.logger.Write(sprintf('CheckAvailableDiskSpace:    disk space available = %0.1f MB,    disk space required = %0.1f MB\n', getFreeDiskSpace()/1e6, obj.MemoryRequired('disk')/1e6));

            if diskspaceToSpare < 0

                msg{1} = sprintf('ERROR: Cannot save processing results requiring %0.1f MB of disk space on current drive with only %0.1f MB of free space available.\n', ...

                                  obj.MemoryRequired('disk')/1e6, getFreeDiskSpace()/1e6);

            elseif diskspacePercentRemaining < 200

                msg{1} = sprintf('WARNING: Available disk space on the current drive is low (%0.1f MB). This may cause problems saving processing results in the future.', ...

                                  getFreeDiskSpace()/1e6);

                msg{2} = sprintf('Consider moving your data set to a drive with more free space\n');

            end            

            if ~isempty(msg)

                MessageBox([msg{:}]);

                obj.logger.Write([msg{:}]);

            end

        end

        

        

            

        % ----------------------------------------------------------------------------------

        function Save(obj)

            fprintf('Saving processed data in %s\n', [obj.path, 'groupResults.mat']);

        function Save(obj, hwait)

            if ~exist('hwait','var')

                hwait = [];

            end            

            

            obj.logger.Write(sprintf('Saving processed data in %s\n', [obj.path, 'groupResults.mat']));

            t_local = tic;

            group = GroupClass(obj);

                       

            % Check that there is anough disk space 

            while CheckAvailableDiskSpace(obj, hwait) < 0

                q = MenuBox('Do you want to save processing results on a different drive?', {'Yes','No'});

                if q==1

                    pname = uigetdir(topLevelDir(), 'Please select alternate folder');

                    if pname == 0

                        return;

                    end

                    obj.pathOutput = pname;

                else

                    return

                end

            end

            

            if ishandle(hwait)

                obj.logger.Write(sprintf('Auto-saving processing results ...\n'), obj.logger.ProgressBar(), hwait);

            end

            

            group = GroupClass(obj); %#ok<NASGU>

            try 

                save([obj.path, 'groupResults.mat'],'group' );

            catch

                msg{1} = sprintf('WARNING: Could not save computation output to groupResults.mat file in the subject folder\n\n');

                msg{2} = sprintf('%s\n\n', pwd);

                msg{3} = sprintf('This could be because the file or the subject folder itself is read-only.');

                msg{4} = sprintf('To fix this change groupResults.mat file write permissions to read/write.');

                MessageBox([msg{:}]);

                save([obj.pathOutput, 'groupResults.mat'],'group' );

            catch ME

                MessageBox(ME.message);

                obj.logger.Write(ME.message);

            end            

            fprintf('Completed saving groupResults.mat in %0.3f seconds.\n', toc(t_local));

            obj.logger.Write(sprintf('Completed saving groupResults.mat in %0.3f seconds.\n', toc(t_local)));

        end
@@ -721,7 +799,7 @@ classdef GroupClass < TreeNodeClass 
        end

        

        % ----------------------------------------------------------------------------------

        function aux = GetAuxiliary(obj)

        function aux = GetAuxiliary(obj) %#ok<MANU>

            aux = [];

        end

DataTree/ProcStream/FuncCallClass.m

+18 −0
Original line number Diff line number Diff line
@@ -581,6 +581,24 @@ classdef FuncCallClass < handle 
            obj.SetUsageName(usagename);

        end

        

        

        % ----------------------------------------------------------------------------------        

        function nbytes = MemoryRequired(obj)            

            fields = properties(obj);

            nbytes = zeros(length(fields),1);

            if isempty(obj)

                nbytes = 0;

                return

            end

            for ii = 1:length(fields)

                fieldstr = sprintf('obj.%s', fields{ii});

                if ~eval('isempty(fieldstr)')

                    nbytes(ii) =  eval(sprintf('sizeof(%s);', fieldstr));

                end

            end

            nbytes = sum(nbytes);

        end

        

    end

DataTree/ProcStream/ProcInputClass.m

+15 −0
Original line number Diff line number Diff line
@@ -113,6 +113,21 @@ classdef ProcInputClass < handle 
            end

        end



        

        % ----------------------------------------------------------------------------------        

        function nbytes = MemoryRequired(obj)

            nbytes(1) = sizeof(obj.tIncMan);

            nbytes(2) = sizeof(obj.mlActMan);

            nbytes(3) = sizeof(obj.misc);

            nbytes(4) = sizeof(obj.stimValSettings);

            if isempty(obj.acquiredEditable)

                nbytes(5) = 0;

            else

                nbytes(5) = obj.acquiredEditable.MemoryRequired();

            end

            nbytes = sum(nbytes);

        end

        

    end

CRA Git | Maintained and supported by SUSTech CRA and CCSE