Loading deepchem/datasets/__init__.py +36 −0 Original line number Diff line number Diff line Loading @@ -70,6 +70,42 @@ class Dataset(object): 'X_sums', 'X_sum_squares', 'X_n', 'y_sums', 'y_sum_squares', 'y_n')) self.save_to_disk() if samples is None and feature_types is not None: # Create an empty metadata dataframe to be filled at a later time basename = "metadata" df_file = "metadata.joblib" out_X = os.path.join(data_dir, "%s-X.joblib" % basename) out_X_transformed = os.path.join(data_dir, "%s-X-transformed.joblib" % basename) out_X_sums = os.path.join(data_dir, "%s-X_sums.joblib" % basename) out_X_sum_squares = os.path.join(data_dir, "%s-X_sum_squares.joblib" % basename) out_X_n = os.path.join(data_dir, "%s-X_n.joblib" % basename) out_y = os.path.join(data_dir, "%s-y.joblib" % basename) out_y_transformed = os.path.join(data_dir, "%s-y-transformed.joblib" % basename) out_y_sums = os.path.join(data_dir, "%s-y_sums.joblib" % basename) out_y_sum_squares = os.path.join(data_dir, "%s-y_sum_squares.joblib" % basename) out_y_n = os.path.join(data_dir, "%s-y_n.joblib" % basename) out_w = os.path.join(data_dir, "%s-w.joblib" % basename) out_ids = os.path.join(data_dir, "%s-ids.joblib" % basename) metadata_rows = [] retval = ([df_file, tasks, out_ids, out_X, out_X_transformed, out_y, out_y_transformed, out_w, out_X_sums, out_X_sum_squares, out_X_n, out_y_sums, out_y_sum_squares, out_y_n]) metadata_rows.append(retval) self.metadata_df = pd.DataFrame( metadata_rows, columns=('df_file','task_names', 'ids', 'X', 'X-transformed', 'y', 'y-transformed', 'w', 'X_sums', 'X_sum_squares', 'X_n', 'y_sums', 'y_sum_squares', 'y_n')) self.save_to_disk() else: if os.path.exists(self._get_metadata_filename()): self.metadata_df = load_from_disk(self._get_metadata_filename()) Loading deepchem/hyperparameters/__init__.py +9 −1 Original line number Diff line number Diff line Loading @@ -15,9 +15,10 @@ class HyperparamOpt(object): Provides simple hyperparameter search capabilities. """ def __init__(self, model_class, task_types, verbosity=None): def __init__(self, model_class, task_types, fit_transformers=None, verbosity=None): self.model_class = model_class self.task_types = task_types self.fit_transformers = fit_transformers assert verbosity in [None, "low", "high"] self.verbosity = verbosity Loading Loading @@ -55,11 +56,18 @@ class HyperparamOpt(object): else: model_dir = tempfile.mkdtemp() if logdir is not None: #TODO(JG) Fit transformers for TF models model = self.model_class(self.task_types, model_params, model_dir, verbosity=self.verbosity) else: if self.fit_transformers: model = self.model_class(self.task_types, model_params, fit_transformers=self.fit_transformers, verbosity=self.verbosity) else: model = self.model_class(self.task_types, model_params, verbosity=self.verbosity) model.fit(train_dataset) model.save(model_dir) Loading deepchem/hyperparameters/tests/test_hyperparam_opt.py +1 −1 Original line number Diff line number Diff line Loading @@ -39,7 +39,7 @@ class TestHyperparamOptAPI(TestAPI): Test hyperparameter optimization API. """ def test_singletask_sklearn_rf_ECFP_regression_hyperparam_opt(self): """Test of singletask RF ECFP regression API.""" """Test of hyperparam_opt with singletask RF ECFP regression API.""" splittype = "scaffold" compound_featurizers = [CircularFingerprint(size=1024)] complex_featurizers = [] Loading deepchem/models/__init__.py +64 −3 Original line number Diff line number Diff line Loading @@ -9,6 +9,7 @@ import numpy as np import pandas as pd import joblib import os import tempfile from deepchem.datasets import Dataset from deepchem.utils.save import load_from_disk from deepchem.utils.save import save_to_disk Loading @@ -26,11 +27,21 @@ class Model(object): Abstract base class for different ML models. """ non_sklearn_models = ["SingleTaskDNN", "MultiTaskDNN", "DockingDNN"] def __init__(self, task_types, model_params, model_instance=None, initialize_raw_model=True, verbosity=None, **kwargs): def __init__(self, task_types, model_params, fit_transformers=None, model_instance=None, initialize_raw_model=True, verbosity=None, **kwargs): self.model_class = model_instance.__class__ self.task_types = task_types self.model_params = model_params self.fit_transformers = fit_transformers if self.fit_transformers: # Initialize batch_dataset self.batch_dataset = self.create_batch_dataset() else: self.batch_dataset = None self.raw_model = None assert verbosity in [None, "low", "high"] self.verbosity = verbosity Loading Loading @@ -91,8 +102,52 @@ class Model(object): batch_size = self.model_params["batch_size"] for epoch in range(self.model_params["nb_epoch"]): log("Starting epoch %s" % str(epoch+1), self.verbosity) losses = [] for (X_batch, y_batch, w_batch, _) in dataset.iterbatches(batch_size): self.fit_on_batch(X_batch, y_batch, w_batch) if self.fit_transformers: X_batch, y_batch, w_batch = self.transform_on_batch(X_batch, y_batch, w_batch, self.batch_dataset) losses.append(self.fit_on_batch(X_batch, y_batch, w_batch)) log("Avg loss for epoch %d: %f" % (epoch+1,np.array(losses).mean()),self.verbosity) def transform_on_batch(self, X, y, w, batch_dataset): """ Transforms data in a 1-shard Dataset object with Transformer objects. """ # Save X, y, and w to batch_dataset # The save/load operations work correctly with 1-shard dataframe df = batch_dataset.metadata_df for _, row in df.iterrows(): save_to_disk(X, row['X-transformed']) save_to_disk(y, row['y-transformed']) save_to_disk(w, row['w']) # Transform batch_dataset for transformer in self.fit_transformers: transformer.transform(batch_dataset) # Return numpy arrays from batch_dataset for _, row in df.iterrows(): X = load_from_disk(row['X-transformed']) y = load_from_disk(row['y-transformed']) w = load_from_disk(row['w']) return X, y, w def create_batch_dataset(self): """ Creates an empty 1-shard Dataset object """ # Create empty dataset data_dir = tempfile.mkdtemp() featurizers = None tasks = self.task_types.keys() batch_dataset = Dataset(data_dir=data_dir, samples=None, featurizers=featurizers, tasks=tasks, use_user_specified_features=True) return batch_dataset # TODO(rbharath): The structure of the produced df might be # complicated. Better way to model? Loading @@ -112,6 +167,12 @@ class Model(object): batch_size = self.model_params["batch_size"] for (X_batch, y_batch, w_batch, ids_batch) in dataset.iterbatches(batch_size): # Apply fit_transformers if needed if self.fit_transformers: X_batch, y_batch, w_batch = self.transform_on_batch(X_batch, y_batch, w_batch, self.batch_dataset) y_pred = self.predict_on_batch(X_batch) y_pred = np.reshape(y_pred, np.shape(y_batch)) Loading deepchem/models/keras_models/fcnet.py +7 −2 Original line number Diff line number Diff line Loading @@ -19,13 +19,17 @@ class MultiTaskDNN(KerasModel): """ Model for multitask MLP in keras. """ def __init__(self, task_types, model_params, def __init__(self, task_types, model_params, fit_transformers=None, initialize_raw_model=True, verbosity="low"): super(MultiTaskDNN, self).__init__(task_types, model_params, fit_transformers=fit_transformers, initialize_raw_model=initialize_raw_model, verbosity=verbosity) if initialize_raw_model: sorted_tasks = sorted(task_types.keys()) if fit_transformers: (n_inputs,) = model_params["init_data_shape"] else: (n_inputs,) = model_params["data_shape"] model = Graph() model.add_input(name="input", input_shape=(n_inputs,)) Loading Loading @@ -136,7 +140,8 @@ class SingleTaskDNN(MultiTaskDNN): """ Abstract base class for different ML models. """ def __init__(self, task_types, model_params, initialize_raw_model=True, verbosity="low"): def __init__(self, task_types, model_params, fit_transformers=None, initialize_raw_model=True, verbosity="low"): super(SingleTaskDNN, self).__init__(task_types, model_params, fit_transformers=fit_transformers, initialize_raw_model=initialize_raw_model, verbosity=verbosity) Loading
deepchem/datasets/__init__.py +36 −0 Original line number Diff line number Diff line Loading @@ -70,6 +70,42 @@ class Dataset(object): 'X_sums', 'X_sum_squares', 'X_n', 'y_sums', 'y_sum_squares', 'y_n')) self.save_to_disk() if samples is None and feature_types is not None: # Create an empty metadata dataframe to be filled at a later time basename = "metadata" df_file = "metadata.joblib" out_X = os.path.join(data_dir, "%s-X.joblib" % basename) out_X_transformed = os.path.join(data_dir, "%s-X-transformed.joblib" % basename) out_X_sums = os.path.join(data_dir, "%s-X_sums.joblib" % basename) out_X_sum_squares = os.path.join(data_dir, "%s-X_sum_squares.joblib" % basename) out_X_n = os.path.join(data_dir, "%s-X_n.joblib" % basename) out_y = os.path.join(data_dir, "%s-y.joblib" % basename) out_y_transformed = os.path.join(data_dir, "%s-y-transformed.joblib" % basename) out_y_sums = os.path.join(data_dir, "%s-y_sums.joblib" % basename) out_y_sum_squares = os.path.join(data_dir, "%s-y_sum_squares.joblib" % basename) out_y_n = os.path.join(data_dir, "%s-y_n.joblib" % basename) out_w = os.path.join(data_dir, "%s-w.joblib" % basename) out_ids = os.path.join(data_dir, "%s-ids.joblib" % basename) metadata_rows = [] retval = ([df_file, tasks, out_ids, out_X, out_X_transformed, out_y, out_y_transformed, out_w, out_X_sums, out_X_sum_squares, out_X_n, out_y_sums, out_y_sum_squares, out_y_n]) metadata_rows.append(retval) self.metadata_df = pd.DataFrame( metadata_rows, columns=('df_file','task_names', 'ids', 'X', 'X-transformed', 'y', 'y-transformed', 'w', 'X_sums', 'X_sum_squares', 'X_n', 'y_sums', 'y_sum_squares', 'y_n')) self.save_to_disk() else: if os.path.exists(self._get_metadata_filename()): self.metadata_df = load_from_disk(self._get_metadata_filename()) Loading
deepchem/hyperparameters/__init__.py +9 −1 Original line number Diff line number Diff line Loading @@ -15,9 +15,10 @@ class HyperparamOpt(object): Provides simple hyperparameter search capabilities. """ def __init__(self, model_class, task_types, verbosity=None): def __init__(self, model_class, task_types, fit_transformers=None, verbosity=None): self.model_class = model_class self.task_types = task_types self.fit_transformers = fit_transformers assert verbosity in [None, "low", "high"] self.verbosity = verbosity Loading Loading @@ -55,11 +56,18 @@ class HyperparamOpt(object): else: model_dir = tempfile.mkdtemp() if logdir is not None: #TODO(JG) Fit transformers for TF models model = self.model_class(self.task_types, model_params, model_dir, verbosity=self.verbosity) else: if self.fit_transformers: model = self.model_class(self.task_types, model_params, fit_transformers=self.fit_transformers, verbosity=self.verbosity) else: model = self.model_class(self.task_types, model_params, verbosity=self.verbosity) model.fit(train_dataset) model.save(model_dir) Loading
deepchem/hyperparameters/tests/test_hyperparam_opt.py +1 −1 Original line number Diff line number Diff line Loading @@ -39,7 +39,7 @@ class TestHyperparamOptAPI(TestAPI): Test hyperparameter optimization API. """ def test_singletask_sklearn_rf_ECFP_regression_hyperparam_opt(self): """Test of singletask RF ECFP regression API.""" """Test of hyperparam_opt with singletask RF ECFP regression API.""" splittype = "scaffold" compound_featurizers = [CircularFingerprint(size=1024)] complex_featurizers = [] Loading
deepchem/models/__init__.py +64 −3 Original line number Diff line number Diff line Loading @@ -9,6 +9,7 @@ import numpy as np import pandas as pd import joblib import os import tempfile from deepchem.datasets import Dataset from deepchem.utils.save import load_from_disk from deepchem.utils.save import save_to_disk Loading @@ -26,11 +27,21 @@ class Model(object): Abstract base class for different ML models. """ non_sklearn_models = ["SingleTaskDNN", "MultiTaskDNN", "DockingDNN"] def __init__(self, task_types, model_params, model_instance=None, initialize_raw_model=True, verbosity=None, **kwargs): def __init__(self, task_types, model_params, fit_transformers=None, model_instance=None, initialize_raw_model=True, verbosity=None, **kwargs): self.model_class = model_instance.__class__ self.task_types = task_types self.model_params = model_params self.fit_transformers = fit_transformers if self.fit_transformers: # Initialize batch_dataset self.batch_dataset = self.create_batch_dataset() else: self.batch_dataset = None self.raw_model = None assert verbosity in [None, "low", "high"] self.verbosity = verbosity Loading Loading @@ -91,8 +102,52 @@ class Model(object): batch_size = self.model_params["batch_size"] for epoch in range(self.model_params["nb_epoch"]): log("Starting epoch %s" % str(epoch+1), self.verbosity) losses = [] for (X_batch, y_batch, w_batch, _) in dataset.iterbatches(batch_size): self.fit_on_batch(X_batch, y_batch, w_batch) if self.fit_transformers: X_batch, y_batch, w_batch = self.transform_on_batch(X_batch, y_batch, w_batch, self.batch_dataset) losses.append(self.fit_on_batch(X_batch, y_batch, w_batch)) log("Avg loss for epoch %d: %f" % (epoch+1,np.array(losses).mean()),self.verbosity) def transform_on_batch(self, X, y, w, batch_dataset): """ Transforms data in a 1-shard Dataset object with Transformer objects. """ # Save X, y, and w to batch_dataset # The save/load operations work correctly with 1-shard dataframe df = batch_dataset.metadata_df for _, row in df.iterrows(): save_to_disk(X, row['X-transformed']) save_to_disk(y, row['y-transformed']) save_to_disk(w, row['w']) # Transform batch_dataset for transformer in self.fit_transformers: transformer.transform(batch_dataset) # Return numpy arrays from batch_dataset for _, row in df.iterrows(): X = load_from_disk(row['X-transformed']) y = load_from_disk(row['y-transformed']) w = load_from_disk(row['w']) return X, y, w def create_batch_dataset(self): """ Creates an empty 1-shard Dataset object """ # Create empty dataset data_dir = tempfile.mkdtemp() featurizers = None tasks = self.task_types.keys() batch_dataset = Dataset(data_dir=data_dir, samples=None, featurizers=featurizers, tasks=tasks, use_user_specified_features=True) return batch_dataset # TODO(rbharath): The structure of the produced df might be # complicated. Better way to model? Loading @@ -112,6 +167,12 @@ class Model(object): batch_size = self.model_params["batch_size"] for (X_batch, y_batch, w_batch, ids_batch) in dataset.iterbatches(batch_size): # Apply fit_transformers if needed if self.fit_transformers: X_batch, y_batch, w_batch = self.transform_on_batch(X_batch, y_batch, w_batch, self.batch_dataset) y_pred = self.predict_on_batch(X_batch) y_pred = np.reshape(y_pred, np.shape(y_batch)) Loading
deepchem/models/keras_models/fcnet.py +7 −2 Original line number Diff line number Diff line Loading @@ -19,13 +19,17 @@ class MultiTaskDNN(KerasModel): """ Model for multitask MLP in keras. """ def __init__(self, task_types, model_params, def __init__(self, task_types, model_params, fit_transformers=None, initialize_raw_model=True, verbosity="low"): super(MultiTaskDNN, self).__init__(task_types, model_params, fit_transformers=fit_transformers, initialize_raw_model=initialize_raw_model, verbosity=verbosity) if initialize_raw_model: sorted_tasks = sorted(task_types.keys()) if fit_transformers: (n_inputs,) = model_params["init_data_shape"] else: (n_inputs,) = model_params["data_shape"] model = Graph() model.add_input(name="input", input_shape=(n_inputs,)) Loading Loading @@ -136,7 +140,8 @@ class SingleTaskDNN(MultiTaskDNN): """ Abstract base class for different ML models. """ def __init__(self, task_types, model_params, initialize_raw_model=True, verbosity="low"): def __init__(self, task_types, model_params, fit_transformers=None, initialize_raw_model=True, verbosity="low"): super(SingleTaskDNN, self).__init__(task_types, model_params, fit_transformers=fit_transformers, initialize_raw_model=initialize_raw_model, verbosity=verbosity)
