Commit 89f095ce authored by Bharath Ramsundar's avatar Bharath Ramsundar Committed by GitHub
Browse files

Merge pull request #262 from peastman/transform 

Transformers work with NumpyDataset
parents 94b91cf6 3e611c52

deepchem/datasets/__init__.py

+84 −35
Original line number Diff line number Diff line
@@ -168,6 +168,28 @@ class Dataset(object): 
    """

    raise NotImplementedError()



  def transform(self, fn, **args):

    """Construct a new dataset by applying a transformation to every sample in this dataset.



    The argument is a function that can be called as follows:



    >>> newx, newy, neww = fn(x, y, w)



    It might be called only once with the whole dataset, or multiple times with different

    subsets of the data.  Each time it is called, it should transform the samples and return

    the transformed data.



    Parameters

    ----------

    fn: function

      A function to apply to each sample in the dataset



    Returns

    -------

    a newly constructed Dataset object

    """

    raise NotImplementedError()



  def get_statistics(self, X_stats=True, y_stats=True):

    """Compute and return statistics of this dataset."""

    X_means = 0.0
@@ -298,6 +320,29 @@ class NumpyDataset(Dataset): 
    n_samples = self._X.shape[0]

    return ((self._X[i], self._y[i], self._w[i], self._ids[i]) for i in range(n_samples))



  def transform(self, fn, **args):

    """Construct a new dataset by applying a transformation to every sample in this dataset.



    The argument is a function that can be called as follows:



    >>> newx, newy, neww = fn(x, y, w)



    It might be called only once with the whole dataset, or multiple times with different

    subsets of the data.  Each time it is called, it should transform the samples and return

    the transformed data.



    Parameters

    ----------

    fn: function

      A function to apply to each sample in the dataset



    Returns

    -------

    a newly constructed Dataset object

    """

    newx, newy, neww = fn(self._X, self._y, self._w)

    return NumpyDataset(newx, newy, neww, self._ids[:], self.verbosity)





class DiskDataset(Dataset):

  """
@@ -559,6 +604,45 @@ class DiskDataset(Dataset): 
                yield (X_shard[i], y_shard[i], w_shard[i], ids_shard[i])

    return iterate(self)



  def transform(self, fn, **args):

    """Construct a new dataset by applying a transformation to every sample in this dataset.



    The argument is a function that can be called as follows:



    >>> newx, newy, neww = fn(x, y, w)



    It might be called only once with the whole dataset, or multiple times with different

    subsets of the data.  Each time it is called, it should transform the samples and return

    the transformed data.



    Parameters

    ----------

    fn: function

      A function to apply to each sample in the dataset

    out_dir: string

      The directory to save the new dataset in.  If this is omitted, a temporary directory

      is created automatically



    Returns

    -------

    a newly constructed Dataset object

    """

    if 'out_dir' in args:

        out_dir = args['out_dir']

    else:

        out_dir = tempfile.mkdtemp()

    tasks = self.get_task_names()

    metadata_rows = []

    for shard_num, row in self.metadata_df.iterrows():

      X, y, w, ids = self.get_shard(shard_num)

      newx, newy, neww = fn(X, y, w)

      basename = "dataset-%d" % shard_num

      metadata_rows.append(DiskDataset.write_data_to_disk(

          out_dir, basename, tasks, newx, newy, neww, ids, False))

    return DiskDataset(data_dir=out_dir,

                   metadata_rows=metadata_rows,

                   verbosity=self.verbosity)



  def reshard(self, shard_size):

    """Reshards data to have specified shard size."""

    # Create temp directory to store resharded version
@@ -940,41 +1024,6 @@ class DiskDataset(Dataset): 
    """Return pandas series of label stds."""

    return self.metadata_df["y_stds"]



  def get_grad_statistics(self):

    """Computes and returns statistics of this dataset



    This function assumes that the first task of a dataset holds the energy for

    an input system, and that the remaining tasks holds the gradient for the

    system.



    TODO(rbharath, joegomes): It is unclear whether this should be a Dataset

    function. Might get refactored out.

    TODO(rbharath, joegomes): If y_n were an exposed part of the API, this

    function could be entirely written in userspace.

    """

    if len(self) == 0:

      return None, None, None, None

    df = self.metadata_df

    y = []

    y_n = []

    for _, row in df.iterrows():

      yy = load_from_disk(os.path.join(self.data_dir, row['y']))

      y.append(yy)

      yn = load_from_disk(os.path.join(self.data_dir, row['y_n']))

      y_n.append(np.array(yn))



    y = np.vstack(y)

    y_n = np.sum(y_n, axis=0)



    energy = y[:,0]

    grad = y[:,1:]

    for i in range(energy.size):

      grad[i] *= energy[i]



    ydely_means = np.sum(grad, axis=0)/y_n[1:]



    return grad, ydely_means



def compute_sums_and_nb_sample(tensor, W=None):

  """

  Computes sums, squared sums of tensor along axis 0.

deepchem/datasets/tests/test_datasets.py

+51 −1
Original line number Diff line number Diff line
@@ -289,7 +289,7 @@ class TestBasicDatasetAPI(TestDatasetAPI): 
        np.testing.assert_array_equal(sw, w[i])

        np.testing.assert_array_equal(sid, ids[i])



  def test_itersamples_dist(self):

  def test_itersamples_disk(self):

    """Test that iterating over samples in a DiskDataset works."""

    solubility_dataset = self.load_solubility_data()

    X = solubility_dataset.X
@@ -302,6 +302,56 @@ class TestBasicDatasetAPI(TestDatasetAPI): 
        np.testing.assert_array_equal(sw, w[i])

        np.testing.assert_array_equal(sid, ids[i])



  def test_transform_numpy(self):

    """Test that the transform() method works for NumpyDatasets."""

    num_datapoints = 100

    num_features = 10

    num_tasks = 10



    # Generate data



    X = np.random.rand(num_datapoints, num_features)

    y = np.random.randint(2, size=(num_datapoints, num_tasks))

    w = np.random.randint(2, size=(num_datapoints, num_tasks))

    ids = np.array(["id"] * num_datapoints)

    dataset = NumpyDataset(X, y, w, ids)



    # Transform it



    def fn(x, y, w):

      return (2*x, 1.5*y, w)

    transformed = dataset.transform(fn)

    np.testing.assert_array_equal(X, dataset.X)

    np.testing.assert_array_equal(y, dataset.y)

    np.testing.assert_array_equal(w, dataset.w)

    np.testing.assert_array_equal(ids, dataset.ids)

    np.testing.assert_array_equal(2*X, transformed.X)

    np.testing.assert_array_equal(1.5*y, transformed.y)

    np.testing.assert_array_equal(w, transformed.w)

    np.testing.assert_array_equal(ids, transformed.ids)



  def test_transform_disk(self):

    """Test that the transform() method works for DiskDatasets."""

    dataset = self.load_solubility_data()

    X = dataset.X

    y = dataset.y

    w = dataset.w

    ids = dataset.ids



    # Transform it



    def fn(x, y, w):

      return (2*x, 1.5*y, w)

    transformed = dataset.transform(fn)

    np.testing.assert_array_equal(X, dataset.X)

    np.testing.assert_array_equal(y, dataset.y)

    np.testing.assert_array_equal(w, dataset.w)

    np.testing.assert_array_equal(ids, dataset.ids)

    np.testing.assert_array_equal(2*X, transformed.X)

    np.testing.assert_array_equal(1.5*y, transformed.y)

    np.testing.assert_array_equal(w, transformed.w)

    np.testing.assert_array_equal(ids, transformed.ids)



  def test_to_numpy(self):

    """Test that transformation to numpy arrays is sensible."""

    solubility_dataset = self.load_solubility_data()

deepchem/datasets/tests/test_reload.py

+1 −1
Original line number Diff line number Diff line
@@ -77,7 +77,7 @@ class TestReload(TestAPI): 
    print("Transforming datasets")

    for dataset in [train_dataset, valid_dataset, test_dataset]:

      for transformer in transformers:

          transformer.transform(dataset)

          dataset = transformer.transform(dataset)



    return (len(train_dataset), len(valid_dataset), len(test_dataset))

deepchem/hyperparameters/tests/test_hyperparam_opt.py

+1 −1
Original line number Diff line number Diff line
@@ -57,7 +57,7 @@ class TestHyperparamOptAPI(TestAPI): 
        NormalizationTransformer(transform_y=True, dataset=train_dataset)]

    for dataset in [train_dataset, test_dataset]:

      for transformer in transformers:

        transformer.transform(dataset)

        dataset = transformer.transform(dataset)



    params_dict = {"n_estimators": [10, 100]}

    metric = Metric(metrics.r2_score)

deepchem/models/tests/test_api.py

+3 −3
Original line number Diff line number Diff line
@@ -108,7 +108,7 @@ class TestModelAPI(TestAPI): 
    transformers = input_transformers + output_transformers

    for dataset in [train_dataset, test_dataset]:

      for transformer in transformers:

        transformer.transform(dataset)

        dataset = transformer.transform(dataset)



    regression_metrics = [Metric(metrics.r2_score),

                          Metric(metrics.mean_squared_error),
@@ -157,7 +157,7 @@ class TestModelAPI(TestAPI): 
    transformers = input_transformers + output_transformers

    for dataset in [train_dataset, test_dataset]:

      for transformer in transformers:

        transformer.transform(dataset)

        dataset = transformer.transform(dataset)



    regression_metrics = [Metric(metrics.r2_score),

                          Metric(metrics.mean_squared_error),
@@ -246,7 +246,7 @@ class TestModelAPI(TestAPI): 


    for dataset in [train_dataset, test_dataset]:

      for transformer in transformers:

        transformer.transform(dataset)

        dataset = transformer.transform(dataset)



    classification_metrics = [Metric(metrics.roc_auc_score),

                              Metric(metrics.matthews_corrcoef),

CRA Git | Maintained and supported by SUSTech CRA and CCSE