Commit 414e32cc authored by Franklin Lee's avatar Franklin Lee
Browse files

CDF and Power X transforms

parent 68fff8cc

deepchem/transformers/__init__.py

+28 −33
Original line number Diff line number Diff line
@@ -68,7 +68,6 @@ class Transformer(object): 
  def transform(self, dataset, parallel=False):

    """

    Transforms all internally stored data.



    Adds X-transform, y-transform columns to metadata.

    """

    df = dataset.metadata_df
@@ -95,7 +94,6 @@ def _transform_row(i, df, transformer, data_dir): 
  """

  Transforms the data (X, y, w,...) in a single row.

  Writes X-transformed, y-transformed to disk.



  """

  transformer.transform_row(i, df, data_dir)
@@ -213,7 +211,8 @@ class AtomicNormalizationTransformer(Transformer): 
    row = df.iloc[i]



    if self.transform_X:

      X = load_from_disk(os.path.join(data_dir, row['X-transformed']))

      X = load_from_disk(

          os.path.join(data_dir, row['X-transformed']))

      X = np.nan_to_num((X - self.X_means) / self.X_stds)

      save_to_disk(X, os.path.join(data_dir, row['X-transformed']))
@@ -446,7 +445,6 @@ class CoulombRandomizationTransformer(Transformer): 
    1. Remove zero padding on Coulomb Matrix

    2. Randomly permute the rows and columns for n_samples

    3. Flatten each sample to upper triangular portion



    Returns list of feature vectors

    """

    max_atom_number = len(cm)
@@ -598,6 +596,7 @@ class CDFTransformer(Transformer): 
    self.transform_X = transform_X

    self.transform_y = transform_y

    self.bins = bins

  # TODO (flee2): for transform_y, figure out weights, untransform



  def transform(self, dataset, bins):

    """Performs CDF transform on data."""
@@ -608,15 +607,18 @@ class CDFTransformer(Transformer): 
      X_t = get_cdf_values(X,self.bins)

      y_t = y

    if self.transform_y:

      print("y will not be transformed by CDFTransformer, for now.")

      """

      y_t = get_cdf_values(y,self.bins)

      X_t = X

      """

    # TODO (rbharath): Find a more elegant solution to saving the data?

    shutil.rmtree(dataset.data_dir)

    os.makedirs(dataset.data_dir)

    DiskDataset.from_numpy(dataset.data_dir, X_t, y_t, w_t, ids_t)



  def untransform(self, z):

    print("Cannot undo CDF Transformer.")

    print("Cannot undo CDF Transformer, for now.")

    # Need this for transform_y



def get_cdf_values(array, bins):
@@ -640,39 +642,33 @@ def get_cdf_values(array, bins): 
  return array_t



class PowerTransformer(Transformer):

  """Takes power n transform of a column and adds it as a new column."""

  """Takes power n transforms of the data based on an input vector."""

  def __init__(self, transform_X=False,

               transform_y=False, features=None, tasks=None,

               n_powers=1, powers=[1]):

               transform_y=False,

               powers=[1]):

    self.transform_X = transform_X

    self.transform_y = transform_y

    self.features = features

    self.tasks = tasks

    self.n_powers = n_powers

    self.powers = powers

    if len(self.powers) != self.n_powers:

      print("Number of powers in list powers is not equal to n_powers.")

      

  def transform(self, dataset, n_powers, powers):

  def transform(self, dataset):

    """Performs power transform on data."""

    X, y, w, ids = (dataset.X, dataset.y, dataset.w, dataset.ids)     

    X_t = X

    y_t = y

    w_t = w

    ids_t = ids

    n_powers = len(self.powers)

    if self.transform_X:

      for i in range(self.n_powers):

	X_temp = np.power(X, self.powers[i])

      	X_t = np.append(X_t, X_temp, axis=1)

        new_features = self.features

        #for j in range(len(self.features)):



      X_t = np.power(X, self.powers[0])

      for i in range(1, n_powers):

      	X_t = np.hstack((X_t,np.power(X, self.powers[i])))

      y_t = y

    if self.transform_y:

      for i in range(self.n_powers):

	y_temp = np.power(y, self.powers[i])

      	y_t = np.append(y_t, y_temp, axis=1)

        new_tasks = self.tasks

        #for j in range(len(self.tasks)):

      print("y will not be transformed by PowerTransformer, for now.")

      """

      y_t = np.power(y, self.powers[0])

      for i in range(1, n_powers):

      	y_t = np.hstack((y_t,np.power(y, self.powers[i])))

      X_t = X

      """



    # TODO (rbharath): Find a more elegant solution to saving the data?

    shutil.rmtree(dataset.data_dir)
@@ -680,9 +676,8 @@ class PowerTransformer(Transformer): 
    DiskDataset.from_numpy(dataset.data_dir, X_t, y_t, w_t, ids_t)



  def untransform(self, z):

    if self.transform_X:

      orig_len = (z.shape[1])/(self.n_powers+1)

      z = z[:,:orig_len]

    if self.transform_y:

    print("Cannot undo Power Transformer, for now.")    

    """

    orig_len = (z.shape[1])/(self.n_powers+1)

    z = z[:,:orig_len]

    """
 
 No newline at end of file

deepchem/transformers/tests/test_transformers.py

+35 −12
Original line number Diff line number Diff line
@@ -14,11 +14,12 @@ import numpy as np 
import pandas as pd

import numpy.random as random

import os

from deepchem.datasets import Dataset

from deepchem.datasets import DiskDataset

from deepchem.transformers import LogTransformer

from deepchem.transformers import NormalizationTransformer

from deepchem.transformers import BalancingTransformer

from deepchem.transformers import CDFTransformer

from deepchem.transformers import PowerTransformer

from deepchem.datasets.tests import TestDatasetAPI



class TestTransformerAPI(TestDatasetAPI):
@@ -191,12 +192,11 @@ class TestTransformerAPI(TestDatasetAPI): 
    target = np.transpose(np.array(np.append([target],[target], axis=0)))

    gaussian_dataset = self.load_gaussian_cdf_data()

    bins=1001

    cdf_transformer = CDFTransformer(

        transform_X=True, bins=bins)

    X, y, w, ids = gaussian_dataset.to_numpy()

    cdf_transformer = CDFTransformer(transform_X=True, bins=bins)

    X, y, w, ids = (gaussian_dataset.X,gaussian_dataset.y,gaussian_dataset.w,gaussian_dataset.ids)

    cdf_transformer.transform(gaussian_dataset, bins=bins)

    gaussian_dataset = Dataset(data_dir=gaussian_dataset.data_dir, reload=True)

    X_t, y_t, w_t, ids_t = gaussian_dataset.to_numpy()

    gaussian_dataset = DiskDataset(data_dir=gaussian_dataset.data_dir,reload=True)

    X_t, y_t, w_t, ids_t = (gaussian_dataset.X,gaussian_dataset.y,gaussian_dataset.w,gaussian_dataset.ids)



    # Check ids are unchanged.

    for id_elt, id_t_elt in zip(ids, ids_t):
@@ -209,18 +209,18 @@ class TestTransformerAPI(TestDatasetAPI): 
    sorted = np.sort(X_t,axis=0)

    np.testing.assert_allclose(sorted, target)



  """

  def test_cdf_y_transformer(self):

    """Test CDF transformer on Gaussian normal dataset."""

    #Test CDF transformer on Gaussian normal dataset.

    target = np.array(np.transpose(np.linspace(0.,1.,1001)))

    target = np.transpose(np.array(np.append([target],[target], axis=0)))

    gaussian_dataset = self.load_gaussian_cdf_data()

    bins=1001

    cdf_transformer = CDFTransformer(

        transform_y=True, bins=bins)

    X, y, w, ids = gaussian_dataset.to_numpy()

    cdf_transformer = CDFTransformer(transform_y=True, bins=bins)

    X, y, w, ids = (gaussian_dataset.X,gaussian_dataset.y,gaussian_dataset.w,gaussian_dataset.ids)

    cdf_transformer.transform(gaussian_dataset, bins=bins)

    gaussian_dataset = Dataset(data_dir=gaussian_dataset.data_dir, reload=True)

    X_t, y_t, w_t, ids_t = gaussian_dataset.to_numpy()

    gaussian_dataset = DiskDataset(data_dir=gaussian_dataset.data_dir,reload=True)

    X_t, y_t, w_t, ids_t = (gaussian_dataset.X,gaussian_dataset.y,gaussian_dataset.w,gaussian_dataset.ids)



    # Check ids are unchanged.

    for id_elt, id_t_elt in zip(ids, ids_t):
@@ -232,6 +232,29 @@ class TestTransformerAPI(TestDatasetAPI): 
    # Check y is now holding the proper values when sorted.

    sorted = np.sort(y_t,axis=0)

    np.testing.assert_allclose(sorted, target)

  """

  

  def test_power_X_transformer(self):

    """Test Power transformer on Gaussian normal dataset."""

    gaussian_dataset = self.load_gaussian_cdf_data()

    powers=[1,2,0.5]

    power_transformer = PowerTransformer(transform_X=True, powers=powers)

    X, y, w, ids = (gaussian_dataset.X,gaussian_dataset.y,gaussian_dataset.w,gaussian_dataset.ids)

    power_transformer.transform(gaussian_dataset)

    gaussian_dataset = DiskDataset(data_dir=gaussian_dataset.data_dir,reload=True)

    X_t, y_t, w_t, ids_t = (gaussian_dataset.X,gaussian_dataset.y,gaussian_dataset.w,gaussian_dataset.ids)



    # Check ids are unchanged.

    for id_elt, id_t_elt in zip(ids, ids_t):

      assert id_elt == id_t_elt

    # Check y is unchanged since this is an X transformer

    np.testing.assert_allclose(y, y_t)

    # Check w is unchanged since this is an X transformer

    np.testing.assert_allclose(w, w_t)

    # Check X is now holding the proper values in each column.

    np.testing.assert_allclose(X, X_t[:,:2])

    np.testing.assert_allclose(np.power(X,2),X_t[:,2:4])

    np.testing.assert_allclose(np.power(X,0.5),X_t[:,4:])

  

  def test_singletask_balancing_transformer(self):

    """Test balancing transformer on single-task dataset."""

CRA Git | Maintained and supported by SUSTech CRA and CCSE