tidying up

      """

      Method that does bulk of splitting dataset appropriately based on desired split percentage

      """

      #finds, for each task, the total number of hits and calculate the required

      #find he total number of hits for each task and calculate the required

      #number of hits for split based on frac_split

      required_hits_list = self.__generate_required_hits(w, frac_split)

      #finds index cutoff in array to get split

      #finds index cutoff per task in array to get required split calculated

      index_list = self.__generate_required_index(w, required_hits_list)

      X_first_split = X_second_split = X

        w_second_split[index:, colIndex] = w[index:, colIndex]

      # check out if any rows in either w_first_split or w_second_split are just zeros

      rowsToKeepTrain = w_first_split.any(axis=1)

      rowsToKeepTest = w_second_split.any(axis=1)

      # prune train sets

      w_first_split = w_first_split[rowsToKeepTrain]

      X_first_split = X_first_split[rowsToKeepTrain]

      y_first_split = y_first_split[rowsToKeepTrain]

      ids_first_split = ids_first_split[rowsToKeepTrain]

      # prune test sets

      w_second_split = w_second_split[rowsToKeepTest]

      X_second_split = X_second_split[rowsToKeepTest]

      y_second_split = y_second_split[rowsToKeepTest]

      ids_second_split = ids_second_split[rowsToKeepTest]

      rowsToKeepFirstSplit = w_first_split.any(axis=1)

      rowsToKeepSecondSplit = w_second_split.any(axis=1)

      # prune first set

      w_first_split = w_first_split[rowsToKeepFirstSplit]

      X_first_split = X_first_split[rowsToKeepFirstSplit]

      y_first_split = y_first_split[rowsToKeepFirstSplit]

      ids_first_split = ids_first_split[rowsToKeepFirstSplit]

      # prune second sets

      w_second_split = w_second_split[rowsToKeepSecondSplit]

      X_second_split = X_second_split[rowsToKeepSecondSplit]

      y_second_split = y_second_split[rowsToKeepSecondSplit]

      ids_second_split = ids_second_split[rowsToKeepSecondSplit]

      return X_first_split, y_first_split, w_first_split, ids_first_split, X_second_split, \

             y_second_split, w_second_split, ids_second_split

                               valid_dir, test_dir, frac_train=.8,

                               frac_valid=.1, frac_test=.1, seed=None,

                               log_every_n=1000):

      # Obtain original x, y, and w arrays and shuffle

      X, y, w, ids = self.__randomize_arrays(dataset.to_numpy())

      print("numpy arrays to be printed, w and y")

      print(w)

      print(y)

      """

      frac_train identifies percentage of datapoints that need to be present in split -- so 80% training data may actually be 90% of data (but 80% of actual datapoints, not NaN, will be present in split)

      """

      X_train, y_train,w_train, ids_train, X_test, y_test, w_test, ids_test = self.__split(X, y, w, ids, frac_train)

      #calculate percent split for valid (out of test and valid)

        colIndex = 0

        for col in w.T:

            if not np.any(col): #check to see if any columns are all zero

                print("good -- at least one column doesn't have results")

                print(colIndex)

                print("weight column")

                print(col)

                print("corresponding y column")

                print(y[:, colIndex])

                sparse_flag = True

                break

            colIndex+=1

        if not sparse_flag:

            print("dataset isn't sparse")

            assert sparse_flag is True

            print("Test dataset isn't sparse -- test failed")

        else:

            print("dataset is sparse")

            print("Column %d is sparse -- expected") % (colIndex)

        assert sparse_flag is True

        stratified_splitter = StratifiedSplitter()

        train_data, valid_data, test_data = \

            # verify that each task in the train dataset has some hits

            for col in w.T:

                if not np.any(col):

                    print("fail -- one column doesn't have results")

                    print("Fail -- one column doesn't have results")

                    if datasetIndex == 0:

                        print("train_data failed")

                    elif datasetIndex == 1: