Commit 615cff21 authored by Bharath Ramsundar's avatar Bharath Ramsundar
Browse files

Changes

parent 5984e9e1

deepchem/data/data_loader.py

+20 −18
Original line number Diff line number Diff line
@@ -79,26 +79,28 @@ def _featurize_smiles_df(df, featurizer, field, log_every_n=1000): 
  """

  sample_elems = df[field].tolist()



  features = []

  from rdkit import Chem

  from rdkit.Chem import rdmolfiles

  from rdkit.Chem import rdmolops

  for ind, elem in enumerate(sample_elems):

    mol = Chem.MolFromSmiles(elem)

    # TODO (ytz) this is a bandage solution to reorder the atoms

    # so that they're always in the same canonical order.

    # Presumably this should be correctly implemented in the

    # future for graph mols.

    if mol:

      new_order = rdmolfiles.CanonicalRankAtoms(mol)

      mol = rdmolops.RenumberAtoms(mol, new_order)

    if ind % log_every_n == 0:

      logger.info("Featurizing sample %d" % ind)

    features.append(featurizer.featurize([mol]))

  features = featurizer(df[field])

  #features = []

  #from rdkit import Chem

  #from rdkit.Chem import rdmolfiles

  #from rdkit.Chem import rdmolops

  #for ind, elem in enumerate(sample_elems):

  #  mol = Chem.MolFromSmiles(elem)

  #  # TODO (ytz) this is a bandage solution to reorder the atoms

  #  # so that they're always in the same canonical order.

  #  # Presumably this should be correctly implemented in the

  #  # future for graph mols.

  #  if mol:

  #    new_order = rdmolfiles.CanonicalRankAtoms(mol)

  #    mol = rdmolops.RenumberAtoms(mol, new_order)

  #  if ind % log_every_n == 0:

  #    logger.info("Featurizing sample %d" % ind)

  #  features.append(featurizer._featurize([mol]))

  valid_inds = np.array(

      [1 if elt.size > 0 else 0 for elt in features], dtype=bool)

  features = [elt for (is_valid, elt) in zip(valid_inds, features) if is_valid]

  return np.squeeze(np.array(features), axis=1), valid_inds

  return np.array(features), valid_inds

  #return np.squeeze(np.array(features), axis=1), valid_inds





def _get_user_specified_features(df, featurizer):
@@ -157,7 +159,7 @@ def _featurize_mol_df(df, featurizer, field, log_every_n=1000): 
  for ind, mol in enumerate(sample_elems):

    if ind % log_every_n == 0:

      logger.info("Featurizing sample %d" % ind)

    features.append(featurizer.featurize([mol]))

    features.append(featurizer._featurize([mol]))

  valid_inds = np.array(

      [1 if elt.size > 0 else 0 for elt in features], dtype=bool)

  features = [elt for (is_valid, elt) in zip(valid_inds, features) if is_valid]

deepchem/data/tests/test_data_loader.py

+35 −41
Original line number Diff line number Diff line 
"""

Tests for FeaturizedSamples class

"""

__author__ = "Bharath Ramsundar"

__copyright__ = "Copyright 2016, Stanford University"

__license__ = "MIT"



import os

import unittest
@@ -12,25 +9,18 @@ import shutil 
import deepchem as dc





class TestDataLoader(unittest.TestCase):

  """

  Test DataLoader

  """



  def setUp(self):

    super(TestDataLoader, self).setUp()

    self.current_dir = os.path.dirname(os.path.abspath(__file__))



  def unlabelled_test(self):

    input_file = os.path.join(self.current_dir,

                              "../../data/tests/no_labels.csv")

def test_unlabelled():

  current_dir = os.path.dirname(os.path.abspath(__file__))

  input_file = os.path.join(current_dir, "../../data/tests/no_labels.csv")

  featurizer = dc.feat.CircularFingerprint(size=1024)

  loader = dc.data.CSVLoader(

      tasks=[], smiles_field="smiles", featurizer=featurizer)

    loader.featurize(input_file)

  loader.create_dataset(input_file)



  def scaffold_test_train_valid_test_split(self):



def scaffold_test_train_valid_test_split():

  """Test of singletask RF ECFP regression API."""

  current_dir = os.path.dirname(os.path.abspath(__file__))

  splittype = "scaffold"

  input_transforms = []

  output_transforms = ["normalize"]
@@ -38,15 +28,14 @@ class TestDataLoader(unittest.TestCase): 
  tasks = ["log-solubility"]

  task_type = "regression"

  task_types = {task: task_type for task in tasks}

    input_file = os.path.join(self.current_dir,

                              "../../models/tests/example.csv")

  input_file = os.path.join(current_dir, "../../models/tests/example.csv")

  featurizer = dc.feat.CircularFingerprint(size=1024)



    input_file = os.path.join(self.current_dir, input_file)

  input_file = os.path.join(current_dir, input_file)

  loader = dc.data.CSVLoader(

      tasks=tasks, smiles_field="smiles", featurizer=featurizer)



    dataset = loader.featurize(input_file)

  dataset = loader.create_dataset(input_file)



  # Splits featurized samples into train/test

  splitter = dc.splits.ScaffoldSplitter()
@@ -56,8 +45,10 @@ class TestDataLoader(unittest.TestCase): 
  assert len(valid_dataset) == 1

  assert len(test_dataset) == 1



  def scaffold_test_train_test_split(self):



def scaffold_test_train_test_split():

  """Test of singletask RF ECFP regression API."""

  current_dir = os.path.dirname(os.path.abspath(__file__))

  splittype = "scaffold"

  input_transforms = []

  output_transforms = ["normalize"]
@@ -65,15 +56,14 @@ class TestDataLoader(unittest.TestCase): 
  tasks = ["log-solubility"]

  task_type = "regression"

  task_types = {task: task_type for task in tasks}

    input_file = os.path.join(self.current_dir,

                              "../../models/tests/example.csv")

  input_file = os.path.join(current_dir, "../../models/tests/example.csv")

  featurizer = dc.feat.CircularFingerprint(size=1024)



    input_file = os.path.join(self.current_dir, input_file)

  input_file = os.path.join(current_dir, input_file)

  loader = dc.data.CSVLoader(

      tasks=tasks, smiles_field="smiles", featurizer=featurizer)



    dataset = loader.featurize(input_file)

  dataset = loader.create_dataset(input_file)



  # Splits featurized samples into train/test

  splitter = dc.splits.ScaffoldSplitter()
@@ -81,23 +71,24 @@ class TestDataLoader(unittest.TestCase): 
  assert len(train_dataset) == 8

  assert len(test_dataset) == 2



  def random_test_train_valid_test_split(self):



def random_test_train_valid_test_split():

  """Test of singletask RF ECFP regression API."""

  current_dir = os.path.dirname(os.path.abspath(__file__))

  input_transforms = []

  output_transforms = ["normalize"]

  model_params = {}

  tasks = ["log-solubility"]

  task_type = "regression"

  task_types = {task: task_type for task in tasks}

    input_file = os.path.join(self.current_dir,

                              "../../models/tests/example.csv")

  input_file = os.path.join(current_dir, "../../models/tests/example.csv")

  featurizer = dc.feat.CircularFingerprint(size=1024)



    input_file = os.path.join(self.current_dir, input_file)

  input_file = os.path.join(current_dir, input_file)

  loader = dc.data.CSVLoader(

      tasks=tasks, smiles_field="smiles", featurizer=featurizer)



    dataset = loader.featurize(input_file)

  dataset = loader.create_dataset(input_file)



  # Splits featurized samples into train/test

  splitter = dc.splits.RandomSplitter()
@@ -107,20 +98,21 @@ class TestDataLoader(unittest.TestCase): 
  assert len(valid_dataset) == 1

  assert len(test_dataset) == 1



  def random_test_train_test_split(self):



def random_test_train_test_split():

  """Test of singletask RF ECFP regression API."""

  current_dir = os.path.dirname(os.path.abspath(__file__))

  #splittype = "random"

  model_params = {}

  tasks = ["log-solubility"]

  task_type = "regression"

  task_types = {task: task_type for task in tasks}

    input_file = os.path.join(self.current_dir,

                              "../../models/tests/example.csv")

  input_file = os.path.join(current_dir, "../../models/tests/example.csv")

  featurizer = dc.feat.CircularFingerprint(size=1024)

  loader = dc.data.CSVLoader(

      tasks=tasks, smiles_field="smiles", featurizer=featurizer)



    dataset = loader.featurize(input_file)

  dataset = loader.create_dataset(input_file)



  # Splits featurized samples into train/test

  splitter = dc.splits.RandomSplitter()
@@ -128,7 +120,8 @@ class TestDataLoader(unittest.TestCase): 
  assert len(train_dataset) == 8

  assert len(test_dataset) == 2



  def test_log_solubility_dataset(self):



def test_log_solubility_dataset():

  """Test of loading for simple log-solubility dataset."""

  current_dir = os.path.dirname(os.path.realpath(__file__))

  input_file = "../../models/tests/example.csv"
@@ -140,23 +133,24 @@ class TestDataLoader(unittest.TestCase): 
      tasks=tasks,

      smiles_field="smiles",

      featurizer=dc.feat.CircularFingerprint(size=1024))

    dataset = loader.featurize(input_file)

  dataset = loader.create_dataset(input_file)



  assert len(dataset) == 10



  def test_dataset_move(self):



def test_dataset_move():

  """Test that dataset can be moved and reloaded."""

  current_dir = os.path.dirname(os.path.abspath(__file__))

  base_dir = tempfile.mkdtemp()

  data_dir = os.path.join(base_dir, "data")

  moved_data_dir = os.path.join(base_dir, "moved_data")

    dataset_file = os.path.join(self.current_dir,

                                "../../models/tests/example.csv")

  dataset_file = os.path.join(current_dir, "../../models/tests/example.csv")



  featurizer = dc.feat.CircularFingerprint(size=1024)

  tasks = ["log-solubility"]

  loader = dc.data.CSVLoader(

      tasks=tasks, smiles_field="smiles", featurizer=featurizer)

    featurized_dataset = loader.featurize(dataset_file, data_dir)

  featurized_dataset = loader.create_dataset(dataset_file, data_dir)

  n_dataset = len(featurized_dataset)



  # Now perform move

CRA Git | Maintained and supported by SUSTech CRA and CCSE