Commit 87992309 authored by Bharath Ramsundar's avatar Bharath Ramsundar
Browse files

Some progress on fixing tests broken by refactor

parent 394bc9df

deepchem/data/__init__.py

+2 −0
Original line number Diff line number Diff line
@@ -14,4 +14,6 @@ from deepchem.data.datasets import DiskDataset 
from deepchem.data.supports import *

from deepchem.data.data_loader import DataLoader

from deepchem.data.data_loader import CSVLoader

from deepchem.data.data_loader import UserCSVLoader

from deepchem.data.data_loader import SDFLoader

import deepchem.data.tests

deepchem/data/data_loader.py

+10 −9
Original line number Diff line number Diff line
@@ -35,7 +35,12 @@ def convert_df_to_numpy(df, tasks, id_field, verbose=False): 
  missing = np.zeros_like(y).astype(int)

  feature_shape = None



  sorted_ids = df[id_field].values

  for ind in range(n_samples):

    for task in range(n_tasks):

      if y[ind, task] == "":

        missing[ind, task] = 1



  ids = df[id_field].values

  # Set missing data to have weight zero

  for ind in range(n_samples):

    for task in range(n_tasks):
@@ -43,8 +48,7 @@ def convert_df_to_numpy(df, tasks, id_field, verbose=False): 
        y[ind, task] = 0.

        w[ind, task] = 0.



  assert len(sorted_ids) == len(y) == len(w)

  return sorted_ids, y.astype(float), w.astype(float)

  return ids, y.astype(float), w.astype(float)



def featurize_smiles_df(df, featurizer, field, log_every_N=1000, verbose=True):

  """Featurize individual compounds in dataframe.
@@ -60,7 +64,7 @@ def featurize_smiles_df(df, featurizer, field, log_every_N=1000, verbose=True): 
    mol = Chem.MolFromSmiles(elem)

    if ind % log_every_N == 0:

      log("Featurizing sample %d" % ind, verbose)

    features.append(featurizer.featurize([mol]))

    features.append(np.squeeze(featurizer.featurize([mol])))

  return np.array(features)



def get_user_specified_features(df, featurizer, verbose=True):
@@ -143,6 +147,7 @@ class DataLoader(object): 
        time1 = time.time()

        X = self.featurize_shard(shard)

        ids, y, w = convert_df_to_numpy(shard, self.tasks, self.id_field)  

        assert len(X) == len(ids) == len(y) == len(w)

        time2 = time.time()

        log("TIMING: featurizing shard %d took %0.3f s" % (shard_num, time2-time1),

            self.verbose)
@@ -167,8 +172,6 @@ class CSVLoader(DataLoader): 


  def featurize_shard(self, shard):

    """Featurizes a shard of an input dataframe."""

    log("Currently featurizing feature_type: %s"

        % self.featurizer.__class__.__name__, self.verbose)

    return featurize_smiles_df(shard, self.featurizer,

                               field=self.smiles_field)

class UserCSVLoader(DataLoader):
@@ -177,12 +180,10 @@ class UserCSVLoader(DataLoader): 
  """

  def get_shards(self, input_files, shard_size):

    """Defines a generator which returns data for each shard"""

    return load_csv_files(input_files, shard_size, verbose=verbose)

    return load_csv_files(input_files, shard_size)



  def featurize_shard(self, shard):

    """Featurizes a shard of an input dataframe."""

    log("Currently featurizing feature_type: %s"

        % self.featurizer.__class__.__name__, self.verbose)

    assert isinstance(self.featurizer, UserDefinedFeaturizer)

    return get_user_specified_features(shard, self.featurizer)

deepchem/data/datasets.py

+8 −16
Original line number Diff line number Diff line
@@ -446,9 +446,9 @@ class DiskDataset(Dataset): 
          y_batch, y_next = y_next[:shard_size], y_next[shard_size:]

          w_batch, w_next = w_next[:shard_size], w_next[shard_size:]

          ids_batch, ids_next = ids_next[:shard_size], ids_next[shard_size:]

          yield (ids_batch, X_batch, y_batch, w_batch)

          yield (X_batch, y_batch, w_batch, ids_batch)

      # Handle spillover from last shard

      yield (ids_next, X_next, y_next, w_next)

      yield (X_next, y_next, w_next, ids_next)

    resharded_dataset = DiskDataset(generator(), data_dir=reshard_dir)

    shutil.rmtree(self.data_dir)

    shutil.move(reshard_dir, self.data_dir)
@@ -595,7 +595,6 @@ class DiskDataset(Dataset): 
    else:

        out_dir = tempfile.mkdtemp()

    tasks = self.get_task_names()

    metadata_rows = []

    def generator():

      for shard_num, row in self.metadata_df.iterrows():

        X, y, w, ids = self.get_shard(shard_num)
@@ -606,8 +605,8 @@ class DiskDataset(Dataset): 
  @staticmethod

  def from_numpy(X, y, w=None, ids=None, tasks=None, data_dir=None):

    """Creates a DiskDataset object from specified Numpy arrays."""

    if data_dir is None:

      data_dir = tempfile.mkdtemp()

    #if data_dir is None:

    #  data_dir = tempfile.mkdtemp()

    n_samples = len(X)

    # The -1 indicates that y will be reshaped to have length -1

    if n_samples > 0:
@@ -621,8 +620,8 @@ class DiskDataset(Dataset): 
      w = np.ones_like(y)

    if tasks is None:

      tasks = np.arange(n_tasks)

    raw_data = (X, y, w, ids)

    return DiskDataset(data_dir=data_dir, tasks=tasks, raw_data=raw_data)

    #raw_data = (X, y, w, ids)

    return DiskDataset([(X, y, w, ids)], data_dir=data_dir, tasks=tasks)



  @staticmethod

  def merge(datasets, merge_dir=None):
@@ -646,8 +645,6 @@ class DiskDataset(Dataset): 
    else:

      subset_dir = tempfile.mkdtemp()

    tasks = self.get_task_names()

    metadata_rows = []

    

    def generator():

      for shard_num, row in self.metadata_df.iterrows():

        if shard_num not in shard_nums:
@@ -662,9 +659,7 @@ class DiskDataset(Dataset): 
    Only for 1-dimensional feature vectors (does not work for tensorial

    featurizations).

    """

    #########################################################  TIMING

    time1 = time.time()

    #########################################################  TIMING

    shard_size = self.get_shard_size()

    num_shards = self.get_number_shards()

    X_sparses, ys, ws, ids = [], [], [], []
@@ -692,11 +687,9 @@ class DiskDataset(Dataset): 
          X_sparse[start:stop], y[start:stop], w[start:stop], ids[start:stop])

      X_s = densify_features(X_sparse_s, num_features)

      self.set_shard(i, X_s, y_s, w_s, ids_s)

    #########################################################  TIMING

    time2 = time.time()

    log("TIMING: sparse_shuffle took %0.3f s" % (time2-time1),

        self.verbose)

    #########################################################  TIMING



  def shuffle_each_shard(self):

    """Shuffles elements within each shard of the datset."""
@@ -760,10 +753,9 @@ class DiskDataset(Dataset): 
    if not len(indices):

      return DiskDataset([], data_dir=select_dir)

    indices = np.array(sorted(indices)).astype(int)

    count, indices_count = 0, 0

    metadata_rows = []

    tasks = self.get_task_names()

    def generator():

      count, indices_count = 0, 0

      for shard_num, (X, y, w, ids) in enumerate(self.itershards()):

        shard_len = len(X)

        # Find indices which rest in this shard
@@ -778,7 +770,7 @@ class DiskDataset(Dataset): 
        y_sel = y[shard_inds]

        w_sel = w[shard_inds]

        ids_sel = ids[shard_inds]

        yield (ids_sel, X_sel, y_sel, w_sel)

        yield (X_sel, y_sel, w_sel, ids_sel)

        # Updating counts

        indices_count += num_shard_elts

        count += shard_len

deepchem/models/tests/test_api.py

+7 −10
Original line number Diff line number Diff line
@@ -28,9 +28,8 @@ class TestAPI(unittest.TestCase): 
    tasks = ["log-solubility"]

    current_dir = os.path.dirname(os.path.abspath(__file__))

    input_file = os.path.join(current_dir, "example.csv")

    loader = dc.data.DataLoader(

        tasks=tasks, smiles_field="smiles",

        featurizer=featurizer)

    loader = dc.data.CSVLoader(

        tasks=tasks, smiles_field="smiles", featurizer=featurizer)

    dataset = loader.featurize(input_file)



    splitter = dc.splits.ScaffoldSplitter()
@@ -61,7 +60,7 @@ class TestAPI(unittest.TestCase): 
    tasks = ["log-solubility"]

    current_dir = os.path.dirname(os.path.abspath(__file__))

    input_file = os.path.join(current_dir, "user_specified_example.csv")

    loader = dc.data.DataLoader(

    loader = dc.data.UserCSVLoader(

        tasks=tasks, smiles_field="smiles", featurizer=featurizer)

    dataset = loader.featurize(input_file)
@@ -97,9 +96,8 @@ class TestAPI(unittest.TestCase): 


    current_dir = os.path.dirname(os.path.abspath(__file__))

    input_file = os.path.join(current_dir, "example.csv")

    loader = dc.data.DataLoader(tasks=tasks,

                        smiles_field="smiles",

                        featurizer=featurizer)

    loader = dc.data.CSVLoader(

        tasks=tasks, smiles_field="smiles", featurizer=featurizer)

    dataset = loader.featurize(input_file)



    splitter = dc.splits.ScaffoldSplitter()
@@ -140,9 +138,8 @@ class TestAPI(unittest.TestCase): 
    current_dir = os.path.dirname(os.path.abspath(__file__))

    input_file = os.path.join(current_dir, "example_classification.csv")



    loader = dc.data.DataLoader(

        tasks=tasks, smiles_field="smiles",

        featurizer=featurizer)

    loader = dc.data.CSVLoader(

        tasks=tasks, smiles_field="smiles", featurizer=featurizer)

    dataset = loader.featurize(input_file)



    splitter = dc.splits.ScaffoldSplitter()

deepchem/models/tests/test_multitask.py

+10 −5
Original line number Diff line number Diff line
@@ -16,12 +16,12 @@ import shutil 
import unittest

import deepchem as dc



class TestMultitaskData(unittest.TestCase):

class TestMultitask(unittest.TestCase):

  """

  Sanity tests for multitask data.

  """

  def setUp(self):

    super(TestMultitaskData, self).setUp()

    super(TestMultitask, self).setUp()

    self.current_dir = os.path.dirname(os.path.abspath(__file__))



  def test_multitask_order(self):
@@ -33,14 +33,19 @@ class TestMultitaskData(unittest.TestCase): 


    featurizer = dc.feat.CircularFingerprint(size=1024)



    loader = dc.data.DataLoader(

        tasks=tasks, smiles_field="smiles",

        featurizer=featurizer)

    loader = dc.data.CSVLoader(

        tasks=tasks, smiles_field="smiles", featurizer=featurizer)

    dataset = loader.featurize(input_file)



    splitter = dc.splits.ScaffoldSplitter()

    train_dataset, test_dataset = splitter.train_test_split(dataset)

  

    ######################################################## DEBUG

    print("train_dataset.get_task_names()")

    print(train_dataset.get_task_names())

    print("tasks")

    print(tasks)

    ######################################################## DEBUG

    assert train_dataset.get_task_names() == tasks

    assert test_dataset.get_task_names() == tasks

CRA Git | Maintained and supported by SUSTech CRA and CCSE