Loading deepchem/models/__init__.py +4 −25 Original line number Diff line number Diff line Loading @@ -34,13 +34,6 @@ class Model(object): self.task_types = task_types self.model_params = model_params self.fit_transformers = fit_transformers if self.fit_transformers: # Initialize batch_dataset self.batch_dataset = self.create_batch_dataset() else: self.batch_dataset = None self.raw_model = None assert verbosity in [None, "low", "high"] Loading Loading @@ -106,32 +99,18 @@ class Model(object): for (X_batch, y_batch, w_batch, _) in dataset.iterbatches(batch_size): if self.fit_transformers: X_batch, y_batch, w_batch = self.transform_on_batch(X_batch, y_batch, w_batch, self.batch_dataset) w_batch) losses.append(self.fit_on_batch(X_batch, y_batch, w_batch)) log("Avg loss for epoch %d: %f" % (epoch+1,np.array(losses).mean()),self.verbosity) def transform_on_batch(self, X, y, w, batch_dataset): def transform_on_batch(self, X, y, w): """ Transforms data in a 1-shard Dataset object with Transformer objects. """ # Save X, y, and w to batch_dataset # The save/load operations work correctly with 1-shard dataframe df = batch_dataset.metadata_df for _, row in df.iterrows(): save_to_disk(X, row['X-transformed']) save_to_disk(y, row['y-transformed']) save_to_disk(w, row['w']) # Transform batch_dataset # Transform X, y, and w for transformer in self.fit_transformers: transformer.transform(batch_dataset) # Return numpy arrays from batch_dataset for _, row in df.iterrows(): X = load_from_disk(row['X-transformed']) y = load_from_disk(row['y-transformed']) w = load_from_disk(row['w']) X, y, w = transformer.transform_on_array(X, y, w) return X, y, w Loading deepchem/transformers/__init__.py +64 −2 Original line number Diff line number Diff line Loading @@ -35,7 +35,11 @@ class Transformer(object): Transforms the data (X, y, w, ...) in a single row). """ raise NotImplementedError( "Each Transformer is responsible for its own tranform_row method.") "Each Transformer is responsible for its own transform_row method.") def transform_array(self, X, y, w): raise NotImplementedError( "Each Transformer is responsible for its own transform_row method.") def untransform(self, z): """Reverses stored transformation on provided data.""" Loading @@ -62,6 +66,20 @@ class Transformer(object): transform_row_partial(index) dataset.save_to_disk() def transform_on_array(self, X, y, w): """ Transforms numpy arrays X, y, and w """ X, y, w = _transform_array(X=X, y=y, w=w, transformer=self) return X, y, w def _transform_array(X, y, w, transformer): """ Performs the numpy array transformation with a given transformer """ X, y, w = transformer.transform_array(X, y, w) return X, y, w def _transform_row(i, df, transformer): """ Transforms the data (X, y, w,...) in a single row. Loading Loading @@ -231,6 +249,20 @@ class CoulombRandomizationTransformer(Transformer): if self.transform_y: print("y will not be transformed by CoulombRandomizationTransformer.") def transform_array(self, X, y, w): """ Randomly permute a Coulomb Matrix passed as an array """ if self.transform_X: for j in xrange(len(X)): cm = self.construct_cm_from_triu(X[j]) X[j] = self.unpad_randomize_and_flatten(cm) if self.transform_y: print("y will not be transformed by CoulombRandomizationTransformer.") return X, y, w def untransform(self, z): print("Cannot undo CoulombRandomizationTransformer.") Loading Loading @@ -272,6 +304,16 @@ class CoulombBinarizationTransformer(Transformer): X_t = (Xt[i]-X_means)/X_stds save_to_disk(X_t, row['X-transformed']) def transform_on_array(self, X, y, w): X, y, w = super(CoulombBinarizationTransformer, self).transform_on_array(X, y, w) X_means = X.mean(axis=0) X_stds = (X-X_means).std() X = (X-X_means)/X_stds return X, y, w def transform_row(self, i, df): """ Binarizes data in dataset with sigmoid function Loading @@ -294,6 +336,26 @@ class CoulombBinarizationTransformer(Transformer): if self.transform_y: print("y will not be transformed by CoulombBinarizationTransformer.") def transform_array(self,X, y, w): """ Binarizes data passed as arrays with sigmoid function """ X_bin = [] if self.update_state: self.set_max(df) self.update_state = False if self.transform_X: for i in range(X.shape[1]): for k in np.arange(0,self.feature_max[i]+self.theta,self.theta): X_bin += [np.tanh((X[:,i]-k)/self.theta)] X = np.array(X_bin).T if self.transform_y: print("y will not be transformed by CoulombBinarizationTransformer.") return X, y, w def untranform(self, z): print("Cannot undo CoulombBinarizationTransformer.") Loading
deepchem/models/__init__.py +4 −25 Original line number Diff line number Diff line Loading @@ -34,13 +34,6 @@ class Model(object): self.task_types = task_types self.model_params = model_params self.fit_transformers = fit_transformers if self.fit_transformers: # Initialize batch_dataset self.batch_dataset = self.create_batch_dataset() else: self.batch_dataset = None self.raw_model = None assert verbosity in [None, "low", "high"] Loading Loading @@ -106,32 +99,18 @@ class Model(object): for (X_batch, y_batch, w_batch, _) in dataset.iterbatches(batch_size): if self.fit_transformers: X_batch, y_batch, w_batch = self.transform_on_batch(X_batch, y_batch, w_batch, self.batch_dataset) w_batch) losses.append(self.fit_on_batch(X_batch, y_batch, w_batch)) log("Avg loss for epoch %d: %f" % (epoch+1,np.array(losses).mean()),self.verbosity) def transform_on_batch(self, X, y, w, batch_dataset): def transform_on_batch(self, X, y, w): """ Transforms data in a 1-shard Dataset object with Transformer objects. """ # Save X, y, and w to batch_dataset # The save/load operations work correctly with 1-shard dataframe df = batch_dataset.metadata_df for _, row in df.iterrows(): save_to_disk(X, row['X-transformed']) save_to_disk(y, row['y-transformed']) save_to_disk(w, row['w']) # Transform batch_dataset # Transform X, y, and w for transformer in self.fit_transformers: transformer.transform(batch_dataset) # Return numpy arrays from batch_dataset for _, row in df.iterrows(): X = load_from_disk(row['X-transformed']) y = load_from_disk(row['y-transformed']) w = load_from_disk(row['w']) X, y, w = transformer.transform_on_array(X, y, w) return X, y, w Loading
deepchem/transformers/__init__.py +64 −2 Original line number Diff line number Diff line Loading @@ -35,7 +35,11 @@ class Transformer(object): Transforms the data (X, y, w, ...) in a single row). """ raise NotImplementedError( "Each Transformer is responsible for its own tranform_row method.") "Each Transformer is responsible for its own transform_row method.") def transform_array(self, X, y, w): raise NotImplementedError( "Each Transformer is responsible for its own transform_row method.") def untransform(self, z): """Reverses stored transformation on provided data.""" Loading @@ -62,6 +66,20 @@ class Transformer(object): transform_row_partial(index) dataset.save_to_disk() def transform_on_array(self, X, y, w): """ Transforms numpy arrays X, y, and w """ X, y, w = _transform_array(X=X, y=y, w=w, transformer=self) return X, y, w def _transform_array(X, y, w, transformer): """ Performs the numpy array transformation with a given transformer """ X, y, w = transformer.transform_array(X, y, w) return X, y, w def _transform_row(i, df, transformer): """ Transforms the data (X, y, w,...) in a single row. Loading Loading @@ -231,6 +249,20 @@ class CoulombRandomizationTransformer(Transformer): if self.transform_y: print("y will not be transformed by CoulombRandomizationTransformer.") def transform_array(self, X, y, w): """ Randomly permute a Coulomb Matrix passed as an array """ if self.transform_X: for j in xrange(len(X)): cm = self.construct_cm_from_triu(X[j]) X[j] = self.unpad_randomize_and_flatten(cm) if self.transform_y: print("y will not be transformed by CoulombRandomizationTransformer.") return X, y, w def untransform(self, z): print("Cannot undo CoulombRandomizationTransformer.") Loading Loading @@ -272,6 +304,16 @@ class CoulombBinarizationTransformer(Transformer): X_t = (Xt[i]-X_means)/X_stds save_to_disk(X_t, row['X-transformed']) def transform_on_array(self, X, y, w): X, y, w = super(CoulombBinarizationTransformer, self).transform_on_array(X, y, w) X_means = X.mean(axis=0) X_stds = (X-X_means).std() X = (X-X_means)/X_stds return X, y, w def transform_row(self, i, df): """ Binarizes data in dataset with sigmoid function Loading @@ -294,6 +336,26 @@ class CoulombBinarizationTransformer(Transformer): if self.transform_y: print("y will not be transformed by CoulombBinarizationTransformer.") def transform_array(self,X, y, w): """ Binarizes data passed as arrays with sigmoid function """ X_bin = [] if self.update_state: self.set_max(df) self.update_state = False if self.transform_X: for i in range(X.shape[1]): for k in np.arange(0,self.feature_max[i]+self.theta,self.theta): X_bin += [np.tanh((X[:,i]-k)/self.theta)] X = np.array(X_bin).T if self.transform_y: print("y will not be transformed by CoulombBinarizationTransformer.") return X, y, w def untranform(self, z): print("Cannot undo CoulombBinarizationTransformer.")
