Commit 8956bc7a authored by Bharath Ramsundar's avatar Bharath Ramsundar Committed by GitHub
Browse files

Merge pull request #239 from flee2/master 

CDF and Power Transformers
parents f5218230 ea56e0e7

deepchem/datasets/tests/__init__.py

+17 −0
Original line number Diff line number Diff line
@@ -107,3 +107,20 @@ class TestDatasetAPI(TestAPI): 
        verbosity="low")

    return loader.featurize(input_file, self.data_dir)



  def load_gaussian_cdf_data(self):

    """Load example with numbers sampled from Gaussian normal distribution.

       Each feature and task is a column of values that is sampled

       from a normal distribution of mean 0, stdev 1."""

    if os.path.exists(self.data_dir):

      shutil.rmtree(self.data_dir)

    features = ["feat0","feat1"]

    featurizer = UserDefinedFeaturizer(features)

    tasks = ["task0","task1"]

    input_file = os.path.join(

        self.current_dir, "../../models/tests/gaussian_cdf_example.csv")

    loader = DataLoader(

        tasks=tasks,

        featurizer=featurizer,

        id_field="id",

        verbosity=None)

    return loader.featurize(input_file, self.data_dir)

deepchem/models/tests/gaussian_cdf_example.csv

0 → 100644
+1002 −0

File added.

Preview size limit exceeded, changes collapsed.

deepchem/transformers/__init__.py

+97 −3
Original line number Diff line number Diff line
@@ -11,6 +11,8 @@ from functools import partial 
from deepchem.utils.save import save_to_disk

from deepchem.utils.save import load_from_disk

from deepchem.utils import pad_array

import shutil

from deepchem.datasets import DiskDataset



def undo_transforms(y, transformers):

  """Undoes all transformations applied."""
@@ -66,7 +68,6 @@ class Transformer(object): 
  def transform(self, dataset, parallel=False):

    """

    Transforms all internally stored data.



    Adds X-transform, y-transform columns to metadata.

    """

    df = dataset.metadata_df
@@ -93,7 +94,6 @@ def _transform_row(i, df, transformer, data_dir): 
  """

  Transforms the data (X, y, w,...) in a single row.

  Writes X-transformed, y-transformed to disk.



  """

  transformer.transform_row(i, df, data_dir)
@@ -445,7 +445,6 @@ class CoulombRandomizationTransformer(Transformer): 
    1. Remove zero padding on Coulomb Matrix

    2. Randomly permute the rows and columns for n_samples

    3. Flatten each sample to upper triangular portion



    Returns list of feature vectors

    """

    max_atom_number = len(cm)
@@ -587,3 +586,98 @@ class CoulombBinarizationTransformer(Transformer): 


  def untranform(self, z):

    print("Cannot undo CoulombBinarizationTransformer.")



class CDFTransformer(Transformer):

  """Histograms the data and assigns values based on sorted list."""

  """Acts like a Cumulative Distribution Function (CDF)."""

  def __init__(self, transform_X=False,

               transform_y=False,

               bins=2):

    self.transform_X = transform_X

    self.transform_y = transform_y

    self.bins = bins

  # TODO (flee2): for transform_y, figure out weights, untransform



  def transform(self, dataset, bins):

    """Performs CDF transform on data."""

    X, y, w, ids = (dataset.X, dataset.y, dataset.w, dataset.ids)  

    w_t = w

    ids_t = ids

    if self.transform_X:

      X_t = get_cdf_values(X,self.bins)

      y_t = y

    if self.transform_y:

      print("y will not be transformed by CDFTransformer, for now.")

      """

      y_t = get_cdf_values(y,self.bins)

      X_t = X

      """

    # TODO (rbharath): Find a more elegant solution to saving the data?

    shutil.rmtree(dataset.data_dir)

    os.makedirs(dataset.data_dir)

    DiskDataset.from_numpy(dataset.data_dir, X_t, y_t, w_t, ids_t)



  def untransform(self, z):

    print("Cannot undo CDF Transformer, for now.")

    # Need this for transform_y



def get_cdf_values(array, bins):

  #array = np.transpose(array)

  n_rows = array.shape[0] 

  n_cols = array.shape[1]

  array_t = np.zeros((n_rows,n_cols))

  parts = n_rows/bins

  hist_values = np.zeros(n_rows)

  sorted_hist_values = np.zeros(n_rows)

  for row in range(n_rows):

    if np.remainder(bins,2)==1:

      hist_values[row] = np.floor(np.divide(row,parts))/(bins-1)

    else:

      hist_values[row] = np.floor(np.divide(row,parts))/bins

  for col in range(n_cols):

    order = np.argsort(array[:,col], axis=0)

    sorted_hist_values = hist_values[order]

    array_t[:,col] = sorted_hist_values

 

  return array_t



class PowerTransformer(Transformer):

  """Takes power n transforms of the data based on an input vector."""

  def __init__(self, transform_X=False,

               transform_y=False,

               powers=[1]):

    self.transform_X = transform_X

    self.transform_y = transform_y

    self.powers = powers

      

  def transform(self, dataset):

    """Performs power transform on data."""

    X, y, w, ids = (dataset.X, dataset.y, dataset.w, dataset.ids)     

    w_t = w

    ids_t = ids

    n_powers = len(self.powers)

    if self.transform_X:

      X_t = np.power(X, self.powers[0])

      for i in range(1, n_powers):

      	X_t = np.hstack((X_t,np.power(X, self.powers[i])))

      y_t = y

    if self.transform_y:

      print("y will not be transformed by PowerTransformer, for now.")

      """

      y_t = np.power(y, self.powers[0])

      for i in range(1, n_powers):

      	y_t = np.hstack((y_t,np.power(y, self.powers[i])))

      X_t = X

      """



    # TODO (rbharath): Find a more elegant solution to saving the data?

    shutil.rmtree(dataset.data_dir)

    os.makedirs(dataset.data_dir)

    DiskDataset.from_numpy(dataset.data_dir, X_t, y_t, w_t, ids_t)



  def untransform(self, z):

    print("Cannot undo Power Transformer, for now.")    

    """

    orig_len = (z.shape[1])/(self.n_powers+1)

    z = z[:,:orig_len]

    """
 
 No newline at end of file

deepchem/transformers/tests/test_transformers.py

+74 −0
Original line number Diff line number Diff line
@@ -12,10 +12,14 @@ __license__ = "GPL" 
import unittest

import numpy as np

import pandas as pd

import numpy.random as random

import os

from deepchem.datasets import DiskDataset

from deepchem.transformers import LogTransformer

from deepchem.transformers import NormalizationTransformer

from deepchem.transformers import BalancingTransformer

from deepchem.transformers import CDFTransformer

from deepchem.transformers import PowerTransformer

from deepchem.datasets.tests import TestDatasetAPI



class TestTransformerAPI(TestDatasetAPI):
@@ -182,6 +186,76 @@ class TestTransformerAPI(TestDatasetAPI): 
    ## Check that untransform does the right thing.

    #np.testing.assert_allclose(normalization_transformer.untransform(X_t), X)



  def test_cdf_X_transformer(self):

    """Test CDF transformer on Gaussian normal dataset."""

    target = np.array(np.transpose(np.linspace(0.,1.,1001)))

    target = np.transpose(np.array(np.append([target],[target], axis=0)))

    gaussian_dataset = self.load_gaussian_cdf_data()

    bins=1001

    cdf_transformer = CDFTransformer(transform_X=True, bins=bins)

    X, y, w, ids = (gaussian_dataset.X,gaussian_dataset.y,gaussian_dataset.w,gaussian_dataset.ids)

    cdf_transformer.transform(gaussian_dataset, bins=bins)

    gaussian_dataset = DiskDataset(data_dir=gaussian_dataset.data_dir,reload=True)

    X_t, y_t, w_t, ids_t = (gaussian_dataset.X,gaussian_dataset.y,gaussian_dataset.w,gaussian_dataset.ids)



    # Check ids are unchanged.

    for id_elt, id_t_elt in zip(ids, ids_t):

      assert id_elt == id_t_elt

    # Check y is unchanged since this is an X transformer

    np.testing.assert_allclose(y, y_t)

    # Check w is unchanged since this is an X transformer

    np.testing.assert_allclose(w, w_t)

    # Check X is now holding the proper values when sorted.

    sorted = np.sort(X_t,axis=0)

    np.testing.assert_allclose(sorted, target)



  """

  def test_cdf_y_transformer(self):

    #Test CDF transformer on Gaussian normal dataset.

    target = np.array(np.transpose(np.linspace(0.,1.,1001)))

    target = np.transpose(np.array(np.append([target],[target], axis=0)))

    gaussian_dataset = self.load_gaussian_cdf_data()

    bins=1001

    cdf_transformer = CDFTransformer(transform_y=True, bins=bins)

    X, y, w, ids = (gaussian_dataset.X,gaussian_dataset.y,gaussian_dataset.w,gaussian_dataset.ids)

    cdf_transformer.transform(gaussian_dataset, bins=bins)

    gaussian_dataset = DiskDataset(data_dir=gaussian_dataset.data_dir,reload=True)

    X_t, y_t, w_t, ids_t = (gaussian_dataset.X,gaussian_dataset.y,gaussian_dataset.w,gaussian_dataset.ids)



    # Check ids are unchanged.

    for id_elt, id_t_elt in zip(ids, ids_t):

      assert id_elt == id_t_elt

    # Check X is unchanged since this is an y transformer

    np.testing.assert_allclose(X, X_t)

    # Check w is unchanged since this is an y transformer

    np.testing.assert_allclose(w, w_t)

    # Check y is now holding the proper values when sorted.

    sorted = np.sort(y_t,axis=0)

    np.testing.assert_allclose(sorted, target)

  """

  

  def test_power_X_transformer(self):

    """Test Power transformer on Gaussian normal dataset."""

    gaussian_dataset = self.load_gaussian_cdf_data()

    powers=[1,2,0.5]

    power_transformer = PowerTransformer(transform_X=True, powers=powers)

    X, y, w, ids = (gaussian_dataset.X,gaussian_dataset.y,gaussian_dataset.w,gaussian_dataset.ids)

    power_transformer.transform(gaussian_dataset)

    gaussian_dataset = DiskDataset(data_dir=gaussian_dataset.data_dir,reload=True)

    X_t, y_t, w_t, ids_t = (gaussian_dataset.X,gaussian_dataset.y,gaussian_dataset.w,gaussian_dataset.ids)



    # Check ids are unchanged.

    for id_elt, id_t_elt in zip(ids, ids_t):

      assert id_elt == id_t_elt

    # Check y is unchanged since this is an X transformer

    np.testing.assert_allclose(y, y_t)

    # Check w is unchanged since this is an X transformer

    np.testing.assert_allclose(w, w_t)

    # Check X is now holding the proper values in each column.

    np.testing.assert_allclose(X, X_t[:,:2])

    np.testing.assert_allclose(np.power(X,2),X_t[:,2:4])

    np.testing.assert_allclose(np.power(X,0.5),X_t[:,4:])

  

  def test_singletask_balancing_transformer(self):

    """Test balancing transformer on single-task dataset."""

CRA Git | Maintained and supported by SUSTech CRA and CCSE