Loading deepchem/datasets/__init__.py +58 −1 Original line number Diff line number Diff line Loading @@ -95,7 +95,8 @@ class Dataset(object): """ if not len(self.metadata_df): raise ValueError("No data in dataset.") sample_X = load_from_disk(self.metadata_df.iterrows().next()[1]['X'])[0] sample_X = load_from_disk( self.metadata_df.iterrows().next()[1]['X-transformed'])[0] return np.shape(sample_X) def _get_metadata_filename(self): Loading Loading @@ -189,6 +190,13 @@ class Dataset(object): X_means, X_stds, y_means, y_stds = compute_mean_and_std(df) return X_means, X_stds, y_means, y_stds def update_moments(self): """Re-compute statistics of this dataset during transformation""" df = self.metadata_df X_means, X_stds, y_means, y_stds = update_mean_and_std(df) return X_means, X_stds, y_means, y_stds def compute_sums_and_nb_sample(tensor, W=None): """ Computes sums, squared sums of tensor along axis 0. Loading Loading @@ -329,3 +337,52 @@ def compute_mean_and_std(df): y_means = np.sum(y_sums, axis=0)/y_n y_vars = np.sum(y_sum_squares, axis=0)/y_n - np.square(y_means) return overall_X_means, np.sqrt(X_vars), y_means, np.sqrt(y_vars) def update_mean_and_std(df): """ Compute means/stds of X/y from sums/sum_squares of tensors. """ X_n = list(df['X_n']) X_transform = [] for _, row in df.iterrows(): Xt = load_from_disk(row['X-transformed']) X_transform.append(np.array(Xt)) # Re-calculate X_sums and X_sum_squares X_sums = [] X_sum_squares = [] for i, row in df.iterrows(): Xs = np.sum(X_transform[i],axis=0) Xss = np.sum(np.square(X_transform[i]),axis=0) X_sums.append(Xs) X_sum_squares.append(Xss) n = float(np.sum(X_n)) X_sums = np.vstack(X_sums) X_sum_squares = np.vstack(X_sum_squares) overall_X_sums = np.sum(X_sums, axis=0) overall_X_means = overall_X_sums / n overall_X_sum_squares = np.sum(X_sum_squares, axis=0) X_vars = (overall_X_sum_squares - np.square(overall_X_sums)/n)/(n) y_n = list(df['y_n']) y_transform = [] for _, row in df.iterrows(): yt = load_from_disk(row['y-transformed']) y_transform.append(np.array(yt)) y_sums = [] y_sum_squares = [] for i, row in df.iterrows(): ys = np.sum(y_transform[i],axis=0) yss = np.sum(np.square(y_transform[i]),axis=0) y_sums.append(ys) y_sum_squares.append(yss) y_n = np.sum(y_n, axis=0) y_sums = np.vstack(y_sums) y_sum_squares = np.vstack(y_sum_squares) y_means = np.sum(y_sums, axis=0)/y_n y_vars = np.sum(y_sum_squares, axis=0)/y_n - np.square(y_means) return overall_X_means, np.sqrt(X_vars), y_means, np.sqrt(y_vars) deepchem/transformers/__init__.py +112 −8 Original line number Diff line number Diff line Loading @@ -10,6 +10,7 @@ import warnings from functools import partial from deepchem.utils.save import save_to_disk from deepchem.utils.save import load_from_disk from deepchem.utils import pad_array # TODO(rbharath): The handling of X/y transforms in the same class is # awkward. Is there a better way to handle this work. Loading Loading @@ -72,11 +73,11 @@ def _transform_row(i, df, transformer): class NormalizationTransformer(Transformer): def __init__(self, transform_X=False, transform_y=False, dataset=None): """Initialize clipping transformation.""" """Initialize normalization transformation.""" super(NormalizationTransformer, self).__init__(transform_X=transform_X, transform_y=transform_y, dataset=dataset) X_means, X_stds, y_means, y_stds = dataset.compute_statistics() X_means, X_stds, y_means, y_stds = dataset.update_moments() self.X_means = X_means self.X_stds = X_stds self.y_means = y_means Loading @@ -89,12 +90,12 @@ class NormalizationTransformer(Transformer): row = df.iloc[i] if self.transform_X: X = load_from_disk(row['X']) X = load_from_disk(row['X-transformed']) X = np.nan_to_num((X - self.X_means) / self.X_stds) save_to_disk(X, row['X-transformed']) if self.transform_y: y = load_from_disk(row['y']) y = load_from_disk(row['y-transformed']) y = np.nan_to_num((y - self.y_means) / self.y_stds) save_to_disk(y, row['y-transformed']) Loading Loading @@ -124,12 +125,12 @@ class ClippingTransformer(Transformer): """ row = df.iloc[i] if self.transform_X: X = load_from_disk(row['X']) X = load_from_disk(row['X-transformed']) X[X > self.max_val] = self.max_val X[X < (-1.0*self.max_val)] = -1.0 * self.max_val save_to_disk(X, row['X-transformed']) if self.transform_y: y = load_from_disk(row['y']) y = load_from_disk(row['y-transformed']) y[y > trunc] = trunc y[y < (-1.0*trunc)] = -1.0 * trunc save_to_disk(y, row['y-transformed']) Loading @@ -144,15 +145,118 @@ class LogTransformer(Transformer): """Logarithmically transforms data in dataset.""" row = df.iloc[i] if self.transform_X: X = load_from_disk(row['X']) X = load_from_disk(row['X-transformed']) X = np.log(X) save_to_disk(X, row['X-transformed']) if self.transform_y: y = load_from_disk(row['y']) y = load_from_disk(row['y-transformed']) y = np.log(y) save_to_disk(y, row['y-transformed']) def untransform(self, z): """Undoes the logarithmic transformation.""" return np.exp(z) class CoulombRandomizationTransformer(Transformer): def __init__(self, transform_X=False, transform_y=False, dataset=None, seed=None): """Iniitialize coulomb matrix randomization transformation. """ super(CoulombRandomizationTransformer, self).__init__(transform_X=transform_X, transform_y=transform_y, dataset=dataset) self.seed = seed def construct_cm_from_triu(self, x): """ Constructs the unpadded coulomb matrix from the upper triangular portion. """ d = int((np.sqrt(8*len(x)+1)-1)/2) cm = np.zeros([d,d]) cm[np.triu_indices_from(cm)] = x for i in xrange(len(cm)): for j in xrange(i+1,len(cm)): cm[j,i] = cm[i,j] return cm def unpad_randomize_and_flatten(self, cm): """ 1. Remove zero padding on Coulomb Matrix 2. Randomly permute the rows and columns for n_samples 3. Flatten each sample to upper triangular portion Returns list of feature vectors """ max_atom_number = len(cm) atom_number = 0 for i in cm[0]: if atom_number == max_atom_number: break elif i != 0.: atom_number += 1 else: break upcm = cm[0:atom_number,0:atom_number] row_norms = np.asarray([np.linalg.norm(row) for row in upcm], dtype=float) rng = np.random.RandomState(self.seed) e = rng.normal(size=row_norms.size) p = np.argsort(row_norms+e) rcm = upcm[p][:,p] rcm = pad_array(rcm, len(cm)) rcm = rcm[np.triu_indices_from(rcm)] return rcm def transform_row(self, i, df): """ Randomly permute a Coulomb Matrix in a dataset """ row = df.iloc[i] if self.transform_X: X = load_from_disk(row['X-transformed']) for j in xrange(len(X)): cm = self.construct_cm_from_triu(X[j]) X[j] = self.unpad_randomize_and_flatten(cm) save_to_disk(X, row['X-transformed']) if self.transform_y: print("y will not be transformed by CoulombRandomizationTransformer.") def untransform(self, z): print("Cannot undo CoulombRandomizationTransformer.") class CoulombBinarizationTransformer(Transformer): def __init__(self, transform_X=False, transform_y=False, dataset=None, theta=1): """Initialize binarization transformation.""" super(CoulombBinarizationTransformer, self).__init__(transform_X=transform_X, transform_y=transform_y, dataset=dataset) self.theta = theta self.max = 2 def transform_row(self, i, df): """ Binarizes data in dataset with sigmoid function """ row = df.iloc[i] X_bin = [] if self.transform_X: X = load_from_disk(row['X-transformed']) d = X[0].shape[0] for i in xrange(len(X)): for j in np.arange(0,self.max+self.theta,self.theta): if j == 0: Xi = np.tanh((X[i])/self.theta).reshape(d,1) else: Xi = np.vstack([Xi,np.tanh((X[i]-j)/self.theta).reshape(d,1)]) X_bin.append(Xi) X_bin = np.array(np.squeeze(X_bin)) save_to_disk(X_bin, row['X-transformed']) if self.transform_y: print("y will not be transformed by CoulombBinarizationTransformer.") def untranform(self, z): print("Cannot undo CoulombBinarizationTransformer.") Loading
deepchem/datasets/__init__.py +58 −1 Original line number Diff line number Diff line Loading @@ -95,7 +95,8 @@ class Dataset(object): """ if not len(self.metadata_df): raise ValueError("No data in dataset.") sample_X = load_from_disk(self.metadata_df.iterrows().next()[1]['X'])[0] sample_X = load_from_disk( self.metadata_df.iterrows().next()[1]['X-transformed'])[0] return np.shape(sample_X) def _get_metadata_filename(self): Loading Loading @@ -189,6 +190,13 @@ class Dataset(object): X_means, X_stds, y_means, y_stds = compute_mean_and_std(df) return X_means, X_stds, y_means, y_stds def update_moments(self): """Re-compute statistics of this dataset during transformation""" df = self.metadata_df X_means, X_stds, y_means, y_stds = update_mean_and_std(df) return X_means, X_stds, y_means, y_stds def compute_sums_and_nb_sample(tensor, W=None): """ Computes sums, squared sums of tensor along axis 0. Loading Loading @@ -329,3 +337,52 @@ def compute_mean_and_std(df): y_means = np.sum(y_sums, axis=0)/y_n y_vars = np.sum(y_sum_squares, axis=0)/y_n - np.square(y_means) return overall_X_means, np.sqrt(X_vars), y_means, np.sqrt(y_vars) def update_mean_and_std(df): """ Compute means/stds of X/y from sums/sum_squares of tensors. """ X_n = list(df['X_n']) X_transform = [] for _, row in df.iterrows(): Xt = load_from_disk(row['X-transformed']) X_transform.append(np.array(Xt)) # Re-calculate X_sums and X_sum_squares X_sums = [] X_sum_squares = [] for i, row in df.iterrows(): Xs = np.sum(X_transform[i],axis=0) Xss = np.sum(np.square(X_transform[i]),axis=0) X_sums.append(Xs) X_sum_squares.append(Xss) n = float(np.sum(X_n)) X_sums = np.vstack(X_sums) X_sum_squares = np.vstack(X_sum_squares) overall_X_sums = np.sum(X_sums, axis=0) overall_X_means = overall_X_sums / n overall_X_sum_squares = np.sum(X_sum_squares, axis=0) X_vars = (overall_X_sum_squares - np.square(overall_X_sums)/n)/(n) y_n = list(df['y_n']) y_transform = [] for _, row in df.iterrows(): yt = load_from_disk(row['y-transformed']) y_transform.append(np.array(yt)) y_sums = [] y_sum_squares = [] for i, row in df.iterrows(): ys = np.sum(y_transform[i],axis=0) yss = np.sum(np.square(y_transform[i]),axis=0) y_sums.append(ys) y_sum_squares.append(yss) y_n = np.sum(y_n, axis=0) y_sums = np.vstack(y_sums) y_sum_squares = np.vstack(y_sum_squares) y_means = np.sum(y_sums, axis=0)/y_n y_vars = np.sum(y_sum_squares, axis=0)/y_n - np.square(y_means) return overall_X_means, np.sqrt(X_vars), y_means, np.sqrt(y_vars)
deepchem/transformers/__init__.py +112 −8 Original line number Diff line number Diff line Loading @@ -10,6 +10,7 @@ import warnings from functools import partial from deepchem.utils.save import save_to_disk from deepchem.utils.save import load_from_disk from deepchem.utils import pad_array # TODO(rbharath): The handling of X/y transforms in the same class is # awkward. Is there a better way to handle this work. Loading Loading @@ -72,11 +73,11 @@ def _transform_row(i, df, transformer): class NormalizationTransformer(Transformer): def __init__(self, transform_X=False, transform_y=False, dataset=None): """Initialize clipping transformation.""" """Initialize normalization transformation.""" super(NormalizationTransformer, self).__init__(transform_X=transform_X, transform_y=transform_y, dataset=dataset) X_means, X_stds, y_means, y_stds = dataset.compute_statistics() X_means, X_stds, y_means, y_stds = dataset.update_moments() self.X_means = X_means self.X_stds = X_stds self.y_means = y_means Loading @@ -89,12 +90,12 @@ class NormalizationTransformer(Transformer): row = df.iloc[i] if self.transform_X: X = load_from_disk(row['X']) X = load_from_disk(row['X-transformed']) X = np.nan_to_num((X - self.X_means) / self.X_stds) save_to_disk(X, row['X-transformed']) if self.transform_y: y = load_from_disk(row['y']) y = load_from_disk(row['y-transformed']) y = np.nan_to_num((y - self.y_means) / self.y_stds) save_to_disk(y, row['y-transformed']) Loading Loading @@ -124,12 +125,12 @@ class ClippingTransformer(Transformer): """ row = df.iloc[i] if self.transform_X: X = load_from_disk(row['X']) X = load_from_disk(row['X-transformed']) X[X > self.max_val] = self.max_val X[X < (-1.0*self.max_val)] = -1.0 * self.max_val save_to_disk(X, row['X-transformed']) if self.transform_y: y = load_from_disk(row['y']) y = load_from_disk(row['y-transformed']) y[y > trunc] = trunc y[y < (-1.0*trunc)] = -1.0 * trunc save_to_disk(y, row['y-transformed']) Loading @@ -144,15 +145,118 @@ class LogTransformer(Transformer): """Logarithmically transforms data in dataset.""" row = df.iloc[i] if self.transform_X: X = load_from_disk(row['X']) X = load_from_disk(row['X-transformed']) X = np.log(X) save_to_disk(X, row['X-transformed']) if self.transform_y: y = load_from_disk(row['y']) y = load_from_disk(row['y-transformed']) y = np.log(y) save_to_disk(y, row['y-transformed']) def untransform(self, z): """Undoes the logarithmic transformation.""" return np.exp(z) class CoulombRandomizationTransformer(Transformer): def __init__(self, transform_X=False, transform_y=False, dataset=None, seed=None): """Iniitialize coulomb matrix randomization transformation. """ super(CoulombRandomizationTransformer, self).__init__(transform_X=transform_X, transform_y=transform_y, dataset=dataset) self.seed = seed def construct_cm_from_triu(self, x): """ Constructs the unpadded coulomb matrix from the upper triangular portion. """ d = int((np.sqrt(8*len(x)+1)-1)/2) cm = np.zeros([d,d]) cm[np.triu_indices_from(cm)] = x for i in xrange(len(cm)): for j in xrange(i+1,len(cm)): cm[j,i] = cm[i,j] return cm def unpad_randomize_and_flatten(self, cm): """ 1. Remove zero padding on Coulomb Matrix 2. Randomly permute the rows and columns for n_samples 3. Flatten each sample to upper triangular portion Returns list of feature vectors """ max_atom_number = len(cm) atom_number = 0 for i in cm[0]: if atom_number == max_atom_number: break elif i != 0.: atom_number += 1 else: break upcm = cm[0:atom_number,0:atom_number] row_norms = np.asarray([np.linalg.norm(row) for row in upcm], dtype=float) rng = np.random.RandomState(self.seed) e = rng.normal(size=row_norms.size) p = np.argsort(row_norms+e) rcm = upcm[p][:,p] rcm = pad_array(rcm, len(cm)) rcm = rcm[np.triu_indices_from(rcm)] return rcm def transform_row(self, i, df): """ Randomly permute a Coulomb Matrix in a dataset """ row = df.iloc[i] if self.transform_X: X = load_from_disk(row['X-transformed']) for j in xrange(len(X)): cm = self.construct_cm_from_triu(X[j]) X[j] = self.unpad_randomize_and_flatten(cm) save_to_disk(X, row['X-transformed']) if self.transform_y: print("y will not be transformed by CoulombRandomizationTransformer.") def untransform(self, z): print("Cannot undo CoulombRandomizationTransformer.") class CoulombBinarizationTransformer(Transformer): def __init__(self, transform_X=False, transform_y=False, dataset=None, theta=1): """Initialize binarization transformation.""" super(CoulombBinarizationTransformer, self).__init__(transform_X=transform_X, transform_y=transform_y, dataset=dataset) self.theta = theta self.max = 2 def transform_row(self, i, df): """ Binarizes data in dataset with sigmoid function """ row = df.iloc[i] X_bin = [] if self.transform_X: X = load_from_disk(row['X-transformed']) d = X[0].shape[0] for i in xrange(len(X)): for j in np.arange(0,self.max+self.theta,self.theta): if j == 0: Xi = np.tanh((X[i])/self.theta).reshape(d,1) else: Xi = np.vstack([Xi,np.tanh((X[i]-j)/self.theta).reshape(d,1)]) X_bin.append(Xi) X_bin = np.array(np.squeeze(X_bin)) save_to_disk(X_bin, row['X-transformed']) if self.transform_y: print("y will not be transformed by CoulombBinarizationTransformer.") def untranform(self, z): print("Cannot undo CoulombBinarizationTransformer.")
