Commit 5a2a9770 authored by Bharath Ramsundar's avatar Bharath Ramsundar
Browse files

First pass at fix for relative paths

parent 0e8c56f5

deepchem/datasets/__init__.py

+149 −111
Original line number Diff line number Diff line
@@ -92,19 +92,34 @@ class Dataset(object): 
        # Create an empty metadata dataframe to be filled at a later time

        basename = "metadata"

        df_file = "metadata.joblib"

        out_X = os.path.join(data_dir, "%s-X.joblib" % basename)

        out_X_transformed = os.path.join(data_dir, "%s-X-transformed.joblib" % basename)

        out_X_sums = os.path.join(data_dir, "%s-X_sums.joblib" % basename)

        out_X_sum_squares = os.path.join(data_dir, "%s-X_sum_squares.joblib" % basename)

        out_X_n = os.path.join(data_dir, "%s-X_n.joblib" % basename)

        out_y = os.path.join(data_dir, "%s-y.joblib" % basename)

        out_y_transformed = os.path.join(data_dir, "%s-y-transformed.joblib" % basename)

        out_y_sums = os.path.join(data_dir, "%s-y_sums.joblib" % basename)

        out_y_sum_squares = os.path.join(data_dir, "%s-y_sum_squares.joblib" % basename)

        out_y_n = os.path.join(data_dir, "%s-y_n.joblib" % basename)

        out_w = os.path.join(data_dir, "%s-w.joblib" % basename)

        out_w_transformed = os.path.join(data_dir, "%s-w-transformed.joblib" % basename)

        out_ids = os.path.join(data_dir, "%s-ids.joblib" % basename)

        ################################# DEBUG

        #out_X = os.path.join(data_dir, "%s-X.joblib" % basename)

        #out_X_transformed = os.path.join(data_dir, "%s-X-transformed.joblib" % basename)

        #out_X_sums = os.path.join(data_dir, "%s-X_sums.joblib" % basename)

        #out_X_sum_squares = os.path.join(data_dir, "%s-X_sum_squares.joblib" % basename)

        #out_X_n = os.path.join(data_dir, "%s-X_n.joblib" % basename)

        #out_y = os.path.join(data_dir, "%s-y.joblib" % basename)

        #out_y_transformed = os.path.join(data_dir, "%s-y-transformed.joblib" % basename)

        #out_y_sums = os.path.join(data_dir, "%s-y_sums.joblib" % basename)

        #out_y_sum_squares = os.path.join(data_dir, "%s-y_sum_squares.joblib" % basename)

        #out_y_n = os.path.join(data_dir, "%s-y_n.joblib" % basename)

        #out_w = os.path.join(data_dir, "%s-w.joblib" % basename)

        #out_w_transformed = os.path.join(data_dir, "%s-w-transformed.joblib" % basename)

        #out_ids = os.path.join(data_dir, "%s-ids.joblib" % basename)

        ################################# DEBUG

        out_X = "%s-X.joblib" % basename

        out_X_transformed = "%s-X-transformed.joblib" % basename

        out_X_sums = "%s-X_sums.joblib" % basename

        out_X_sum_squares = "%s-X_sum_squares.joblib" % basename

        out_X_n = "%s-X_n.joblib" % basename

        out_y = "%s-y.joblib" % basename

        out_y_transformed = "%s-y-transformed.joblib" % basename

        out_y_sums = "%s-y_sums.joblib" % basename

        out_y_sum_squares = "%s-y_sum_squares.joblib" % basename

        out_y_n = "%s-y_n.joblib" % basename

        out_w = "%s-w.joblib" % basename

        out_w_transformed = "%s-w-transformed.joblib" % basename

        out_ids = "%s-ids.joblib" % basename



        metadata_rows = []

        retval = ([df_file, tasks, out_ids,
@@ -151,7 +166,9 @@ class Dataset(object): 
    if not len(self.metadata_df):

      raise ValueError("No data in dataset.")

    sample_X = load_from_disk(

        self.metadata_df.iterrows().next()[1]['X-transformed'])[0]

        os.path.join(

            self.data_dir,

            self.metadata_df.iterrows().next()[1]['X-transformed']))[0]

    return np.shape(sample_X)



  def _get_metadata_filename(self):
@@ -176,10 +193,14 @@ class Dataset(object): 
    The order of shards returned is guaranteed to remain fixed.

    """

    for _, row in self.metadata_df.iterrows():

      X = np.array(load_from_disk(row['X-transformed']))

      y = np.array(load_from_disk(row['y-transformed']))

      w = np.array(load_from_disk(row['w-transformed']))

      ids = np.array(load_from_disk(row['ids']), dtype=object)

      X = np.array(load_from_disk(

          os.path.join(self.data_dir, row['X-transformed'])))

      y = np.array(load_from_disk(

          os.path.join(self.data_dir, row['y-transformed'])))

      w = np.array(load_from_disk(

          os.path.join(self.data_dir, row['w-transformed'])))

      ids = np.array(load_from_disk(

          os.path.join(self.data_dir, row['ids'])), dtype=object)

      yield (X, y, w, ids)



  def iterbatches(self, batch_size=None, epoch=0):
@@ -267,7 +288,7 @@ class Dataset(object): 
    """

    total = 0

    for _, row in self.metadata_df.iterrows():

      y = load_from_disk(row['y-transformed'])

      y = load_from_disk(os.path.join(self.data_dir, row['y-transformed']))

      total += len(y)

    return total
@@ -288,11 +309,78 @@ class Dataset(object): 
    X_means, X_stds, y_means, y_stds = compute_mean_and_std(df)

    return X_means, X_stds, y_means, y_stds



  def _compute_mean_and_std(self, df):

    """

    Compute means/stds of X/y from sums/sum_squares of tensors.

    """



    X_sums = []

    X_sum_squares = []

    X_n = []

    for _, row in df.iterrows():

      Xs = load_from_disk(os.path.join(self.data_dir, row['X_sums']))

      Xss = load_from_disk(os.path.join(self.data_dir, row['X_sum_squares']))

      Xn = load_from_disk(os.path.join(self.data_dir, row['X_n']))

      X_sums.append(np.array(Xs))

      X_sum_squares.append(np.array(Xss))

      X_n.append(np.array(Xn))



    # Note that X_n is a list of floats

    n = float(np.sum(X_n))

    X_sums = np.vstack(X_sums)

    X_sum_squares = np.vstack(X_sum_squares)

    overall_X_sums = np.sum(X_sums, axis=0)

    overall_X_means = overall_X_sums / n

    overall_X_sum_squares = np.sum(X_sum_squares, axis=0)



    X_vars = (overall_X_sum_squares - np.square(overall_X_sums)/n)/(n)



    y_sums = []

    y_sum_squares = []

    y_n = []

    for _, row in df.iterrows():

      ys = load_from_disk(os.path.join(self.data_dir, row['y_sums']))

      yss = load_from_disk(os.path.join(self.data_dir, row['y_sum_squares']))

      yn = load_from_disk(os.path.join(self.data_dir, row['y_n']))

      y_sums.append(np.array(ys))

      y_sum_squares.append(np.array(yss))

      y_n.append(np.array(yn))



    # Note y_n is a list of arrays of shape (n_tasks,)

    y_n = np.sum(y_n, axis=0)

    y_sums = np.vstack(y_sums)

    y_sum_squares = np.vstack(y_sum_squares)

    y_means = np.sum(y_sums, axis=0)/y_n

    y_vars = np.sum(y_sum_squares, axis=0)/y_n - np.square(y_means)

    return overall_X_means, np.sqrt(X_vars), y_means, np.sqrt(y_vars)



  

  def update_moments(self):

    """Re-compute statistics of this dataset during transformation"""

    df = self.metadata_df

    update_mean_and_std(df)



  def _update_mean_and_std(self, df):

    """

    Compute means/stds of X/y from sums/sum_squares of tensors.

    """

    X_transform = []

    for _, row in df.iterrows():

      Xt = load_from_disk(os.path.join(self.data_dir, row['X-transformed']))

      Xs = np.sum(Xt,axis=0)

      Xss = np.sum(np.square(Xt),axis=0)

      save_to_disk(Xs, os.path.join(self.data_dir, row['X_sums']))

      save_to_disk(Xss, os.path.join(self.data_dir, row['X_sum_squares']))



    y_transform = []

    for _, row in df.iterrows():

      yt = load_from_disk(os.path.join(self.data_dir, row['y-transformed']))

      ys = np.sum(yt,axis=0)

      yss = np.sum(np.square(yt),axis=0)

      save_to_disk(ys, os.path.join(self.data_dir, row['y_sums']))

      save_to_disk(yss, os.path.join(self.data_dir, row['y_sum_squares']))



 



def compute_sums_and_nb_sample(tensor, W=None):

  """
@@ -346,35 +434,50 @@ def write_dataset_single(val, data_dir, feature_types=None, tasks=None, 


  if feature_types is not None and tasks is not None:

    basename = os.path.splitext(os.path.basename(df_file))[0]

  out_X = os.path.join(data_dir, "%s-X.joblib" % basename)

  out_X_transformed = os.path.join(data_dir, "%s-X-transformed.joblib" % basename)

  out_X_sums = os.path.join(data_dir, "%s-X_sums.joblib" % basename)

  out_X_sum_squares = os.path.join(data_dir, "%s-X_sum_squares.joblib" % basename)

  out_X_n = os.path.join(data_dir, "%s-X_n.joblib" % basename)

  out_y = os.path.join(data_dir, "%s-y.joblib" % basename)

  out_y_transformed = os.path.join(data_dir, "%s-y-transformed.joblib" % basename)

  out_y_sums = os.path.join(data_dir, "%s-y_sums.joblib" % basename)

  out_y_sum_squares = os.path.join(data_dir, "%s-y_sum_squares.joblib" % basename)

  out_y_n = os.path.join(data_dir, "%s-y_n.joblib" % basename)

  out_w = os.path.join(data_dir, "%s-w.joblib" % basename)

  out_w_transformed = os.path.join(data_dir, "%s-w-transformed.joblib" % basename)

  out_ids = os.path.join(data_dir, "%s-ids.joblib" % basename)



  save_to_disk(X, out_X)

  save_to_disk(y, out_y)

  save_to_disk(w, out_w)

  ######################################### DEBUG

  #out_X = os.path.join(data_dir, "%s-X.joblib" % basename)

  #out_X_transformed = os.path.join(data_dir, "%s-X-transformed.joblib" % basename)

  #out_X_sums = os.path.join(data_dir, "%s-X_sums.joblib" % basename)

  #out_X_sum_squares = os.path.join(data_dir, "%s-X_sum_squares.joblib" % basename)

  #out_X_n = os.path.join(data_dir, "%s-X_n.joblib" % basename)

  #out_y = os.path.join(data_dir, "%s-y.joblib" % basename)

  #out_y_transformed = os.path.join(data_dir, "%s-y-transformed.joblib" % basename)

  #out_y_sums = os.path.join(data_dir, "%s-y_sums.joblib" % basename)

  #out_y_sum_squares = os.path.join(data_dir, "%s-y_sum_squares.joblib" % basename)

  #out_y_n = os.path.join(data_dir, "%s-y_n.joblib" % basename)

  #out_w = os.path.join(data_dir, "%s-w.joblib" % basename)

  #out_w_transformed = os.path.join(data_dir, "%s-w-transformed.joblib" % basename)

  #out_ids = os.path.join(data_dir, "%s-ids.joblib" % basename)

  ######################################### DEBUG

  out_X = "%s-X.joblib" % basename

  out_X_transformed = "%s-X-transformed.joblib" % basename

  out_X_sums = "%s-X_sums.joblib" % basename

  out_X_sum_squares = "%s-X_sum_squares.joblib" % basename

  out_X_n = "%s-X_n.joblib" % basename

  out_y = "%s-y.joblib" % basename

  out_y_transformed = "%s-y-transformed.joblib" % basename

  out_y_sums = "%s-y_sums.joblib" % basename

  out_y_sum_squares = "%s-y_sum_squares.joblib" % basename

  out_y_n = "%s-y_n.joblib" % basename

  out_w = "%s-w.joblib" % basename

  out_w_transformed = "%s-w-transformed.joblib" % basename

  out_ids = "%s-ids.joblib" % basename



  save_to_disk(X, os.path.join(data_dir, out_X))

  save_to_disk(y, os.path.join(data_dir, out_y))

  save_to_disk(w, os.path.join(data_dir, out_w))

  # Write moments to disk

  save_to_disk(X_sums, out_X_sums)

  save_to_disk(X_sum_squares, out_X_sum_squares)

  save_to_disk(X_n, out_X_n)

  save_to_disk(y_sums, out_y_sums)

  save_to_disk(y_sum_squares, out_y_sum_squares)

  save_to_disk(y_n, out_y_n)

  save_to_disk(X_sums, os.path.join(data_dir, out_X_sums))

  save_to_disk(X_sum_squares, os.path.join(data_dir, out_X_sum_squares))

  save_to_disk(X_n, os.path.join(data_dir, out_X_n))

  save_to_disk(y_sums, os.path.join(data_dir, out_y_sums))

  save_to_disk(y_sum_squares, os.path.join(data_dir, out_y_sum_squares))

  save_to_disk(y_n, os.path.join(data_dir, out_y_n))

  # Write X, y as transformed versions

  save_to_disk(X, out_X_transformed)

  save_to_disk(y, out_y_transformed)

  save_to_disk(w, out_w_transformed)

  save_to_disk(ids, out_ids)

  save_to_disk(X, os.path.join(data_dir, out_X_transformed))

  save_to_disk(y, os.path.join(data_dir, out_y_transformed))

  save_to_disk(w, os.path.join(data_dir, out_w_transformed))

  save_to_disk(ids, os.path.join(data_dir, out_ids))

  return([df_file, tasks, out_ids, out_X, out_X_transformed, out_y,

          out_y_transformed, out_w, out_w_transformed,

          out_X_sums, out_X_sum_squares, out_X_n,
@@ -438,68 +541,3 @@ def _df_to_numpy(df, feature_types, tasks): 
  assert len(sorted_ids) == len(x) == len(y) == len(w)

  return sorted_ids, x.astype(float), y.astype(float), w.astype(float)

def compute_mean_and_std(df):

  """

  Compute means/stds of X/y from sums/sum_squares of tensors.

  """



  X_sums = []

  X_sum_squares = []

  X_n = []

  for _, row in df.iterrows():

    Xs = load_from_disk(row['X_sums'])

    Xss = load_from_disk(row['X_sum_squares'])

    Xn = load_from_disk(row['X_n'])

    X_sums.append(np.array(Xs))

    X_sum_squares.append(np.array(Xss))

    X_n.append(np.array(Xn))



  # Note that X_n is a list of floats

  n = float(np.sum(X_n))

  X_sums = np.vstack(X_sums)

  X_sum_squares = np.vstack(X_sum_squares)

  overall_X_sums = np.sum(X_sums, axis=0)

  overall_X_means = overall_X_sums / n

  overall_X_sum_squares = np.sum(X_sum_squares, axis=0)



  X_vars = (overall_X_sum_squares - np.square(overall_X_sums)/n)/(n)



  y_sums = []

  y_sum_squares = []

  y_n = []

  for _, row in df.iterrows():

    ys = load_from_disk(row['y_sums'])

    yss = load_from_disk(row['y_sum_squares'])

    yn = load_from_disk(row['y_n'])

    y_sums.append(np.array(ys))

    y_sum_squares.append(np.array(yss))

    y_n.append(np.array(yn))



  # Note y_n is a list of arrays of shape (n_tasks,)

  y_n = np.sum(y_n, axis=0)

  y_sums = np.vstack(y_sums)

  y_sum_squares = np.vstack(y_sum_squares)

  y_means = np.sum(y_sums, axis=0)/y_n

  y_vars = np.sum(y_sum_squares, axis=0)/y_n - np.square(y_means)

  return overall_X_means, np.sqrt(X_vars), y_means, np.sqrt(y_vars)



def update_mean_and_std(df):

  """

  Compute means/stds of X/y from sums/sum_squares of tensors.

  """

  X_transform = []

  for _, row in df.iterrows():

    Xt = load_from_disk(row['X-transformed'])

    Xs = np.sum(Xt,axis=0)

    Xss = np.sum(np.square(Xt),axis=0)

    save_to_disk(Xs, row['X_sums'])

    save_to_disk(Xss, row['X_sum_squares'])



  y_transform = []

  for _, row in df.iterrows():

    yt = load_from_disk(row['y-transformed'])

    ys = np.sum(yt,axis=0)

    yss = np.sum(np.square(yt),axis=0)

    save_to_disk(ys, row['y_sums'])

    save_to_disk(yss, row['y_sum_squares'])

deepchem/datasets/tests/test_load.py

+1 −1
Original line number Diff line number Diff line
@@ -67,7 +67,7 @@ class TestLoad(TestAPI): 
    np.testing.assert_allclose(X, X_moved)

    np.testing.assert_allclose(y, y_moved)

    np.testing.assert_allclose(w, w_moved)

    np.testing.assert_allclose(ids, ids_moved)

    np.testing.assert_array_equal(ids, ids_moved)

CRA Git | Maintained and supported by SUSTech CRA and CCSE