Loading deepchem/models/__init__.py +22 −28 Original line number Diff line number Diff line Loading @@ -37,13 +37,6 @@ class Model(object): self.task_types = task_types self.model_params = model_params self.fit_transformers = fit_transformers if self.fit_transformers: # Initialize batch_dataset self.batch_dataset = self.create_batch_dataset() else: self.batch_dataset = None self.raw_model = None assert verbosity in [None, "low", "high"] Loading Loading @@ -109,32 +102,18 @@ class Model(object): for (X_batch, y_batch, w_batch, _) in dataset.iterbatches(batch_size): if self.fit_transformers: X_batch, y_batch, w_batch = self.transform_on_batch(X_batch, y_batch, w_batch, self.batch_dataset) w_batch) losses.append(self.fit_on_batch(X_batch, y_batch, w_batch)) log("Avg loss for epoch %d: %f" % (epoch+1,np.array(losses).mean()),self.verbosity) def transform_on_batch(self, X, y, w, batch_dataset): def transform_on_batch(self, X, y, w): """ Transforms data in a 1-shard Dataset object with Transformer objects. """ # Save X, y, and w to batch_dataset # The save/load operations work correctly with 1-shard dataframe df = batch_dataset.metadata_df for _, row in df.iterrows(): save_to_disk(X, row['X-transformed']) save_to_disk(y, row['y-transformed']) save_to_disk(w, row['w']) # Transform batch_dataset # Transform X, y, and w for transformer in self.fit_transformers: transformer.transform(batch_dataset) # Return numpy arrays from batch_dataset for _, row in df.iterrows(): X = load_from_disk(row['X-transformed']) y = load_from_disk(row['y-transformed']) w = load_from_disk(row['w']) X, y, w = transformer.transform_on_array(X, y, w) return X, y, w Loading Loading @@ -171,15 +150,30 @@ class Model(object): batch_size = self.model_params["batch_size"] for (X_batch, y_batch, w_batch, ids_batch) in dataset.iterbatches(batch_size): # Apply fit_transformers if needed # HACK(JG): This was a hack to perform n-fold averaging of y_pred on # a given X_batch. If fit_transformers exist, we will apply them to # X_batch 1 times and average the resulting y_pred before we undo # transforms on y_pred and y. In the future the averaging will be # performed n_sample times, where n_sample can be user-specified. if self.fit_transformers: X_batch, y_batch, w_batch = self.transform_on_batch(X_batch, y_batch, w_batch, self.batch_dataset) y_preds = [] for i in xrange(1): X_b, y_b, w_b = self.transform_on_batch(X_batch, y_batch, w_batch) y_pred = self.predict_on_batch(X_b) y_pred = np.reshape(y_pred, np.shape(y_b)) y_preds.append(y_pred) y_pred = np.array(y_preds).mean(axis=0) else: y_pred = self.predict_on_batch(X_batch) y_pred = np.reshape(y_pred, np.shape(y_batch)) # Now undo transformations on y, y_pred y_raw, y_pred_raw = y_batch, y_pred y_batch = undo_transforms(y_batch, transformers) y_pred = undo_transforms(y_pred, transformers) Loading deepchem/models/keras_models/fcnet.py +0 −3 Original line number Diff line number Diff line Loading @@ -27,9 +27,6 @@ class MultiTaskDNN(KerasModel): verbosity=verbosity) if initialize_raw_model: sorted_tasks = sorted(task_types.keys()) if fit_transformers: (n_inputs,) = model_params["init_data_shape"] else: (n_inputs,) = model_params["data_shape"] model = Graph() model.add_input(name="input", input_shape=(n_inputs,)) Loading deepchem/splits/__init__.py +31 −0 Original line number Diff line number Diff line Loading @@ -117,6 +117,37 @@ class Splitter(object): """ raise NotImplementedError class MolecularWeightSplitter(Splitter): """ Class for doing data splits by molecular weight. """ def split(self, samples, seed=None, frac_train=.8, frac_valid=.1, frac_test=.1, log_every_n=None): """ Splits internal compounds into train/validation/test using the MW calculated by SMILES string. """ np.testing.assert_almost_equal(frac_train + frac_valid + frac_test, 1.) np.random.seed(seed) df = samples.compounds_df mws = [] for _, row in df.iterrows(): mol = Chem.MolFromSmiles(row['smiles']) mw = Chem.rdMolDescriptors.CalcExactMolWt(mol) mws.append(mw) # Sort by increasing MW mws = np.array(mws) sortidx = np.argsort(mws) train_cutoff = frac_train * len(sortidx) valid_cutoff = (frac_train+frac_valid) * len(sortidx) return (sortidx[:train_cutoff], sortidx[train_cutoff:valid_cutoff], sortidx[valid_cutoff:]) class RandomSplitter(Splitter): """ Class for doing random data splits. Loading deepchem/transformers/__init__.py +50 −1 Original line number Diff line number Diff line Loading @@ -36,7 +36,11 @@ class Transformer(object): Transforms the data (X, y, w, ...) in a single row). """ raise NotImplementedError( "Each Transformer is responsible for its own tranform_row method.") "Each Transformer is responsible for its own transform_row method.") def transform_array(self, X, y, w): raise NotImplementedError( "Each Transformer is responsible for its own transform_array method.") def untransform(self, z): """Reverses stored transformation on provided data.""" Loading Loading @@ -64,6 +68,13 @@ class Transformer(object): transform_row_partial(index) dataset.save_to_disk() def transform_on_array(self, X, y, w): """ Transforms numpy arrays X, y, and w """ X, y, w = self.transform_array(X, y, w) return X, y, w def _transform_row(i, df, transformer): """ Transforms the data (X, y, w,...) in a single row. Loading Loading @@ -286,6 +297,20 @@ class CoulombRandomizationTransformer(Transformer): print("y will not be transformed by " "CoulombRandomizationTransformer.") def transform_array(self, X, y, w): """ Randomly permute a Coulomb Matrix passed as an array """ if self.transform_X: for j in xrange(len(X)): cm = self.construct_cm_from_triu(X[j]) X[j] = self.unpad_randomize_and_flatten(cm) if self.transform_y: print("y will not be transformed by CoulombRandomizationTransformer.") return X, y, w def untransform(self, z): print("Cannot undo CoulombRandomizationTransformer.") Loading Loading @@ -349,5 +374,29 @@ class CoulombBinarizationTransformer(Transformer): print("y will not be transformed by " "CoulombBinarizationTransformer.") def transform_array(self,X, y, w): """ Binarizes data passed as arrays with sigmoid function """ X_bin = [] if self.update_state: self.set_max(df) self.update_state = False if self.transform_X: for i in range(X.shape[1]): for k in np.arange(0,self.feature_max[i]+self.theta,self.theta): X_bin += [np.tanh((X[:,i]-k)/self.theta)] X = np.array(X_bin).T X_means = X.mean(axis=0) X_stds = (X-X_means).std() X = (X-X_means)/X_stds if self.transform_y: print("y will not be transformed by CoulombBinarizationTransformer.") return X, y, w def untranform(self, z): print("Cannot undo CoulombBinarizationTransformer.") Loading
deepchem/models/__init__.py +22 −28 Original line number Diff line number Diff line Loading @@ -37,13 +37,6 @@ class Model(object): self.task_types = task_types self.model_params = model_params self.fit_transformers = fit_transformers if self.fit_transformers: # Initialize batch_dataset self.batch_dataset = self.create_batch_dataset() else: self.batch_dataset = None self.raw_model = None assert verbosity in [None, "low", "high"] Loading Loading @@ -109,32 +102,18 @@ class Model(object): for (X_batch, y_batch, w_batch, _) in dataset.iterbatches(batch_size): if self.fit_transformers: X_batch, y_batch, w_batch = self.transform_on_batch(X_batch, y_batch, w_batch, self.batch_dataset) w_batch) losses.append(self.fit_on_batch(X_batch, y_batch, w_batch)) log("Avg loss for epoch %d: %f" % (epoch+1,np.array(losses).mean()),self.verbosity) def transform_on_batch(self, X, y, w, batch_dataset): def transform_on_batch(self, X, y, w): """ Transforms data in a 1-shard Dataset object with Transformer objects. """ # Save X, y, and w to batch_dataset # The save/load operations work correctly with 1-shard dataframe df = batch_dataset.metadata_df for _, row in df.iterrows(): save_to_disk(X, row['X-transformed']) save_to_disk(y, row['y-transformed']) save_to_disk(w, row['w']) # Transform batch_dataset # Transform X, y, and w for transformer in self.fit_transformers: transformer.transform(batch_dataset) # Return numpy arrays from batch_dataset for _, row in df.iterrows(): X = load_from_disk(row['X-transformed']) y = load_from_disk(row['y-transformed']) w = load_from_disk(row['w']) X, y, w = transformer.transform_on_array(X, y, w) return X, y, w Loading Loading @@ -171,15 +150,30 @@ class Model(object): batch_size = self.model_params["batch_size"] for (X_batch, y_batch, w_batch, ids_batch) in dataset.iterbatches(batch_size): # Apply fit_transformers if needed # HACK(JG): This was a hack to perform n-fold averaging of y_pred on # a given X_batch. If fit_transformers exist, we will apply them to # X_batch 1 times and average the resulting y_pred before we undo # transforms on y_pred and y. In the future the averaging will be # performed n_sample times, where n_sample can be user-specified. if self.fit_transformers: X_batch, y_batch, w_batch = self.transform_on_batch(X_batch, y_batch, w_batch, self.batch_dataset) y_preds = [] for i in xrange(1): X_b, y_b, w_b = self.transform_on_batch(X_batch, y_batch, w_batch) y_pred = self.predict_on_batch(X_b) y_pred = np.reshape(y_pred, np.shape(y_b)) y_preds.append(y_pred) y_pred = np.array(y_preds).mean(axis=0) else: y_pred = self.predict_on_batch(X_batch) y_pred = np.reshape(y_pred, np.shape(y_batch)) # Now undo transformations on y, y_pred y_raw, y_pred_raw = y_batch, y_pred y_batch = undo_transforms(y_batch, transformers) y_pred = undo_transforms(y_pred, transformers) Loading
deepchem/models/keras_models/fcnet.py +0 −3 Original line number Diff line number Diff line Loading @@ -27,9 +27,6 @@ class MultiTaskDNN(KerasModel): verbosity=verbosity) if initialize_raw_model: sorted_tasks = sorted(task_types.keys()) if fit_transformers: (n_inputs,) = model_params["init_data_shape"] else: (n_inputs,) = model_params["data_shape"] model = Graph() model.add_input(name="input", input_shape=(n_inputs,)) Loading
deepchem/splits/__init__.py +31 −0 Original line number Diff line number Diff line Loading @@ -117,6 +117,37 @@ class Splitter(object): """ raise NotImplementedError class MolecularWeightSplitter(Splitter): """ Class for doing data splits by molecular weight. """ def split(self, samples, seed=None, frac_train=.8, frac_valid=.1, frac_test=.1, log_every_n=None): """ Splits internal compounds into train/validation/test using the MW calculated by SMILES string. """ np.testing.assert_almost_equal(frac_train + frac_valid + frac_test, 1.) np.random.seed(seed) df = samples.compounds_df mws = [] for _, row in df.iterrows(): mol = Chem.MolFromSmiles(row['smiles']) mw = Chem.rdMolDescriptors.CalcExactMolWt(mol) mws.append(mw) # Sort by increasing MW mws = np.array(mws) sortidx = np.argsort(mws) train_cutoff = frac_train * len(sortidx) valid_cutoff = (frac_train+frac_valid) * len(sortidx) return (sortidx[:train_cutoff], sortidx[train_cutoff:valid_cutoff], sortidx[valid_cutoff:]) class RandomSplitter(Splitter): """ Class for doing random data splits. Loading
deepchem/transformers/__init__.py +50 −1 Original line number Diff line number Diff line Loading @@ -36,7 +36,11 @@ class Transformer(object): Transforms the data (X, y, w, ...) in a single row). """ raise NotImplementedError( "Each Transformer is responsible for its own tranform_row method.") "Each Transformer is responsible for its own transform_row method.") def transform_array(self, X, y, w): raise NotImplementedError( "Each Transformer is responsible for its own transform_array method.") def untransform(self, z): """Reverses stored transformation on provided data.""" Loading Loading @@ -64,6 +68,13 @@ class Transformer(object): transform_row_partial(index) dataset.save_to_disk() def transform_on_array(self, X, y, w): """ Transforms numpy arrays X, y, and w """ X, y, w = self.transform_array(X, y, w) return X, y, w def _transform_row(i, df, transformer): """ Transforms the data (X, y, w,...) in a single row. Loading Loading @@ -286,6 +297,20 @@ class CoulombRandomizationTransformer(Transformer): print("y will not be transformed by " "CoulombRandomizationTransformer.") def transform_array(self, X, y, w): """ Randomly permute a Coulomb Matrix passed as an array """ if self.transform_X: for j in xrange(len(X)): cm = self.construct_cm_from_triu(X[j]) X[j] = self.unpad_randomize_and_flatten(cm) if self.transform_y: print("y will not be transformed by CoulombRandomizationTransformer.") return X, y, w def untransform(self, z): print("Cannot undo CoulombRandomizationTransformer.") Loading Loading @@ -349,5 +374,29 @@ class CoulombBinarizationTransformer(Transformer): print("y will not be transformed by " "CoulombBinarizationTransformer.") def transform_array(self,X, y, w): """ Binarizes data passed as arrays with sigmoid function """ X_bin = [] if self.update_state: self.set_max(df) self.update_state = False if self.transform_X: for i in range(X.shape[1]): for k in np.arange(0,self.feature_max[i]+self.theta,self.theta): X_bin += [np.tanh((X[:,i]-k)/self.theta)] X = np.array(X_bin).T X_means = X.mean(axis=0) X_stds = (X-X_means).std() X = (X-X_means)/X_stds if self.transform_y: print("y will not be transformed by CoulombBinarizationTransformer.") return X, y, w def untranform(self, z): print("Cannot undo CoulombBinarizationTransformer.")
