Commit be914af5 authored by Bharath Ramsundar's avatar Bharath Ramsundar
Browse files

Fix tricky error with transformers

parent 745ce419

deepchem/datasets/__init__.py

+0 −15
Original line number Diff line number Diff line
@@ -434,21 +434,6 @@ def write_dataset_single(val, data_dir, feature_types=None, tasks=None, 


  if feature_types is not None and tasks is not None:

    basename = os.path.splitext(os.path.basename(df_file))[0]

  ######################################### DEBUG

  #out_X = os.path.join(data_dir, "%s-X.joblib" % basename)

  #out_X_transformed = os.path.join(data_dir, "%s-X-transformed.joblib" % basename)

  #out_X_sums = os.path.join(data_dir, "%s-X_sums.joblib" % basename)

  #out_X_sum_squares = os.path.join(data_dir, "%s-X_sum_squares.joblib" % basename)

  #out_X_n = os.path.join(data_dir, "%s-X_n.joblib" % basename)

  #out_y = os.path.join(data_dir, "%s-y.joblib" % basename)

  #out_y_transformed = os.path.join(data_dir, "%s-y-transformed.joblib" % basename)

  #out_y_sums = os.path.join(data_dir, "%s-y_sums.joblib" % basename)

  #out_y_sum_squares = os.path.join(data_dir, "%s-y_sum_squares.joblib" % basename)

  #out_y_n = os.path.join(data_dir, "%s-y_n.joblib" % basename)

  #out_w = os.path.join(data_dir, "%s-w.joblib" % basename)

  #out_w_transformed = os.path.join(data_dir, "%s-w-transformed.joblib" % basename)

  #out_ids = os.path.join(data_dir, "%s-ids.joblib" % basename)

  ######################################### DEBUG

  out_X = "%s-X.joblib" % basename

  out_X_transformed = "%s-X-transformed.joblib" % basename

  out_X_sums = "%s-X_sums.joblib" % basename

deepchem/models/tests/test_generalize.py

+1 −0
Original line number Diff line number Diff line
@@ -95,6 +95,7 @@ class TestGeneralization(TestAPI): 
    train_dataset = Dataset.from_numpy(self.train_dir, X_train, y_train)

    test_dataset = Dataset.from_numpy(self.test_dir, X_test, y_test)





    # Eval model on train

    input_transformers = [

        NormalizationTransformer(transform_X=True, dataset=train_dataset),

deepchem/transformers/__init__.py

+35 −38
Original line number Diff line number Diff line
@@ -35,13 +35,12 @@ class Transformer(object): 
    self.transform_X = transform_X

    self.transform_y = transform_y

    self.transform_w = transform_w

    self.data_dir = dataset.data_dir

    # One, but not both, transform_X or tranform_y is true

    assert transform_X or transform_y or transform_w

    # Use fact that bools add as ints in python

    assert (transform_X + transform_y + transform_w) == 1 



  def transform_row(self, i, df):

  def transform_row(self, i, df, data_dir):

    """

    Transforms the data (X, y, w, ...) in a single row).

    """
@@ -68,7 +67,7 @@ class Transformer(object): 
    df = dataset.metadata_df

    indices = range(0, df.shape[0])

    transform_row_partial = partial(

        _transform_row, df=df, transformer=self)

        _transform_row, df=df, transformer=self, data_dir=dataset.data_dir)

    if parallel:

      pool = mp.Pool(int(mp.cpu_count()/4))

      pool.map(transform_row_partial, indices)
@@ -85,13 +84,13 @@ class Transformer(object): 
    X, y, w = self.transform_array(X, y, w)    

    return X, y, w



def _transform_row(i, df, transformer):

def _transform_row(i, df, transformer, data_dir):

  """

  Transforms the data (X, y, w,...) in a single row.



  Writes X-transforme,d y-transformed to disk.

  """

  transformer.transform_row(i, df)

  transformer.transform_row(i, df, data_dir)



class NormalizationTransformer(Transformer):
@@ -112,7 +111,7 @@ class NormalizationTransformer(Transformer): 
        dataset, parallel=parallel)

    



  def transform_row(self, i, df):

  def transform_row(self, i, df, data_dir):

    """

    Normalizes the data (X, y, w, ...) in a single row).

    """
@@ -120,14 +119,14 @@ class NormalizationTransformer(Transformer): 


    if self.transform_X:

      X = load_from_disk(

          os.path.join(self.data_dir, row['X-transformed']))

          os.path.join(data_dir, row['X-transformed']))

      X = np.nan_to_num((X - self.X_means) / self.X_stds)

      save_to_disk(X, os.path.join(self.data_dir, row['X-transformed']))

      save_to_disk(X, os.path.join(data_dir, row['X-transformed']))



    if self.transform_y:

      y = load_from_disk(os.path.join(self.data_dir, row['y-transformed']))

      y = load_from_disk(os.path.join(data_dir, row['y-transformed']))

      y = np.nan_to_num((y - self.y_means) / self.y_stds)

      save_to_disk(y, os.path.join(self.data_dir, row['y-transformed']))

      save_to_disk(y, os.path.join(data_dir, row['y-transformed']))



  def untransform(self, z):

    """
@@ -136,7 +135,6 @@ class NormalizationTransformer(Transformer): 
    if self.transform_X:

      return z * self.X_stds + self.X_means

    elif self.transform_y:

      out = z * self.y_stds + self.y_means

      return z * self.y_stds + self.y_means



class ClippingTransformer(Transformer):
@@ -149,23 +147,22 @@ class ClippingTransformer(Transformer): 
                                              transform_w=transform_w,

                                              dataset=dataset)

    self.max_val = max_val

    self.data_dir = dataset.data_dir



  def transform_row(self, i, df):

  def transform_row(self, i, df, data_dir):

    """

    Clips outliers for the data (X, y, w, ...) in a single row).

    """

    row = df.iloc[i]

    if self.transform_X:

      X = load_from_disk(os.path.join(self.data_dir, row['X-transformed']))

      X = load_from_disk(os.path.join(data_dir, row['X-transformed']))

      X[X > self.max_val] = self.max_val

      X[X < (-1.0*self.max_val)] = -1.0 * self.max_val

      save_to_disk(X, os.path.join(self.data_dir, row['X-transformed']))

      save_to_disk(X, os.path.join(data_dir, row['X-transformed']))

    if self.transform_y:

      y = load_from_disk(os.path.join(self.data_dir, row['y-transformed']))

      y = load_from_disk(os.path.join(data_dir, row['y-transformed']))

      y[y > trunc] = trunc

      y[y < (-1.0*trunc)] = -1.0 * trunc

      save_to_disk(y, os.path.join(self.data_dir, row['y-transformed']))

      save_to_disk(y, os.path.join(data_dir, row['y-transformed']))



  def untransform(self, z):

    warnings.warn("Clipping cannot be undone.")
@@ -173,18 +170,18 @@ class ClippingTransformer(Transformer): 


class LogTransformer(Transformer):



  def transform_row(self, i, df):

  def transform_row(self, i, df, data_dir):

    """Logarithmically transforms data in dataset."""

    row = df.iloc[i]

    if self.transform_X:

      X = load_from_disk(os.path.join(self.data_dir, row['X-transformed']))

      X = load_from_disk(os.path.join(data_dir, row['X-transformed']))

      X = np.log(X)

      save_to_disk(X, os.path.join(self.data_dir, row['X-transformed']))

      save_to_disk(X, os.path.join(data_dir, row['X-transformed']))



    if self.transform_y:

      y = load_from_disk(os.path.join(self.data_dir, row['y-transformed']))

      y = load_from_disk(os.path.join(data_dir, row['y-transformed']))

      y = np.log(y)

      save_to_disk(y, os.path.join(self.data_dir, row['y-transformed']))

      save_to_disk(y, os.path.join(data_dir, row['y-transformed']))



  def untransform(self, z):

    """Undoes the logarithmic transformation."""
@@ -223,11 +220,11 @@ class BalancingTransformer(Transformer): 
      weights.append((neg_weight, pos_weight))

    self.weights = weights



  def transform_row(self, i, df):

  def transform_row(self, i, df, data_dir):

    """Reweight the labels for this data."""

    row = df.iloc[i]

    y = load_from_disk(os.path.join(self.data_dir, row['y-transformed']))

    w = load_from_disk(os.path.join(self.data_dir, row['w-transformed']))

    y = load_from_disk(os.path.join(data_dir, row['y-transformed']))

    w = load_from_disk(os.path.join(data_dir, row['w-transformed']))

    w_balanced = np.zeros_like(w)

    for ind, task in enumerate(self.dataset.get_task_names()):

      task_y = y[:, ind]
@@ -236,7 +233,7 @@ class BalancingTransformer(Transformer): 
      one_indices = np.logical_and(task_y==1, task_w != 0)

      w_balanced[zero_indices, ind] = self.weights[ind][0]

      w_balanced[one_indices, ind] = self.weights[ind][1]

    save_to_disk(w_balanced, os.path.join(self.data_dir, row['w-transformed']))

    save_to_disk(w_balanced, os.path.join(data_dir, row['w-transformed']))



class CoulombRandomizationTransformer(Transformer):
@@ -288,17 +285,17 @@ class CoulombRandomizationTransformer(Transformer): 


    return rcm



  def transform_row(self, i, df):

  def transform_row(self, i, df, data_dir):

    """

    Randomly permute a Coulomb Matrix in a dataset

    """

    row = df.iloc[i]

    if self.transform_X:

      X = load_from_disk(os.path.join(self.data_dir, row['X-transformed']))

      X = load_from_disk(os.path.join(data_dir, row['X-transformed']))

      for j in xrange(len(X)):

        cm = self.construct_cm_from_triu(X[j])

        X[j] = self.unpad_randomize_and_flatten(cm)

      save_to_disk(X, os.path.join(self.data_dir, row['X-transformed']))

      save_to_disk(X, os.path.join(data_dir, row['X-transformed']))



    if self.transform_y:

      print("y will not be transformed by "
@@ -333,10 +330,10 @@ class CoulombBinarizationTransformer(Transformer): 
    self.feature_max = np.zeros(dataset.get_data_shape())

    self.update_state = update_state



  def set_max(self, df):

  def set_max(self, df, data_dir):

    

    for _, row in df.iterrows(): 

      X = load_from_disk(os.path.join(self.data_dir, row['X-transformed']))

      X = load_from_disk(os.path.join(data_dir, row['X-transformed']))

      self.feature_max = np.maximum(self.feature_max,X.max(axis=0))



  def transform(self, dataset, parallel=False):
@@ -348,7 +345,7 @@ class CoulombBinarizationTransformer(Transformer): 
    Xt = []



    for _, row in df.iterrows():

      X_t = load_from_disk(os.path.join(self.data_dir, row['X-transformed']))

      X_t = load_from_disk(os.path.join(dataset.data_dir, row['X-transformed']))

      Xt.append(np.array(X_t))



    X = np.vstack(Xt)
@@ -357,25 +354,25 @@ class CoulombBinarizationTransformer(Transformer): 


    for i, row in df.iterrows():

      X_t = (Xt[i]-X_means)/X_stds

      save_to_disk(X_t, os.path.join(self.data_dir, row['X-transformed']))

      save_to_disk(X_t, os.path.join(dataset.data_dir, row['X-transformed']))



  def transform_row(self, i, df):

  def transform_row(self, i, df, data_dir):

    """

    Binarizes data in dataset with sigmoid function

    """

    row = df.iloc[i]

    X_bin = []

    if self.update_state: 

      self.set_max(df)

      self.set_max(df, data_dir)

      self.update_state = False

    if self.transform_X:

      X = load_from_disk(os.path.join(self.data_dir, row['X-transformed']))

      X = load_from_disk(os.path.join(data_dir, row['X-transformed']))

      for i in range(X.shape[1]):

        for k in np.arange(0,self.feature_max[i]+self.theta,self.theta):

          X_bin += [np.tanh((X[:,i]-k)/self.theta)]



      X_bin = np.array(X_bin).T

      save_to_disk(X_bin, os.path.join(self.data_dir, row['X-transformed']))

      save_to_disk(X_bin, os.path.join(data_dir, row['X-transformed']))



    if self.transform_y:

      print("y will not be transformed by "
@@ -388,7 +385,7 @@ class CoulombBinarizationTransformer(Transformer): 


    X_bin = []

    if self.update_state: 

      self.set_max(df)

      self.set_max(df, data_dir)

      self.update_state = False

    if self.transform_X:

      for i in range(X.shape[1]):

CRA Git | Maintained and supported by SUSTech CRA and CCSE