Commit d00f63c7 authored by Bharath Ramsundar's avatar Bharath Ramsundar
Browse files

Add tests for proper dropping of unparseable compounds

parent a88b70d0

deepchem/datasets/__init__.py

+5 −0
Original line number Diff line number Diff line
@@ -183,6 +183,11 @@ class Dataset(object): 
    Returns minibatches from dataset.

    """

    for i, (X, y, w, ids) in enumerate(self.itershards()):

      ######################## DEBUG

      print("Dataset.iterbatches()")

      print("i, X.shape, y.shape")

      print(i, X.shape, y.shape)

      ######################## DEBUG

      nb_sample = np.shape(X)[0]

      if batch_size is None:

        shard_batch_size = nb_sample

deepchem/datasets/tests/mini_emols.csv

0 → 100644
+26 −0
Original line number Diff line number Diff line 
smiles,id,activity

O=C1CCc2c(N1)[c-]c([c-][c-]2)OCCCC[N+]1([O-])CCN(CC1)c1[c-][c-][c-]c(c1Cl)Cl,48866084_50429806,0

O=C1CCc2c(N1)[c-]c([c-][c-]2)OCCCCN1CC[N+](CC1)([O-])c1[c-][c-][c-]c(c1Cl)Cl,48866086_50429808,0

CO[C@H]1O[C@H]2O[C@]3(C)CC[C@H]4[C@@]2([C@@H]([C@H]1C)CC[C@@H]4C)OO3,48866088_48866087,0

O=C1O[C@@H]2O[C@]3(C)CC[C@H]4[C@@]2([C@H]([C@@H]1C)CC[C@@H]4C)OO3,48866090_48866089,0

O=C1O[C@@H]2O[C@]3(C)CC[C@H]4[C@@]2([C@H](C1=C)CC[C@@H]4C)OO3,48866092_48866091,0

OCC1O[C@@H](O[C@@H]2C[C@@H](C(=O)O)[C@@H]3[C@](C2)(C)[C@@H]2CC[C@@H]4C[C@@]2(CC3)[C@@H](O)C4=C)C(C([C@@H]1OS(=O)(=O)[O-])OS(=O)(=O)[O-])OC(=O)CC(C)C.[Na+].[Na+],48866104_48866103,0

OC1C[C@@H](O[C@@H]1COP(=O)(O)O)n1cnc(nc1=O)N,48866106_48866105,0

C/C=C(/C(=O)OC1C[C@H](OC(=O)C)C2([C@@H]3[C@@]41CO[C@@]([C@H]4[C@@](C)([C@H]([C@H]3OC2)O)[C@@]12OC2(C)C2CC1O[C@@H]1C2(O)C=CO1)(O)C(=O)OC)C(=O)OC)\C,48866108_48866107,0

CN1CCC(=C2c3[c-][c-][c-][c-]c3CCc3c2n[c-][c-][c-]3)CC1.OC(=O)/C=C\C(=O)O,48866111_33542275,0

Clc1[c-][c-]c([c-][c-]1)Cc1nn(C2CCC[N+](CC2)([O-])C)c(=O)c2c1[c-][c-][c-][c-]2,48866115_48866114,0

CC[C@@H]1OC(=O)[C@H](C)[C@H](OC2OC(C)C(C(C2)(C)OC)O)[C@@H](C)[C@H](OC2OC(C)CC(C2O)[N+](C)(C)[O-])[C@](C[C@@H](CN([C@@H]([C@H](C1(C)O)O)C)C)C)(C)O,48866130_48866129,0

CO/C=C(\c1[c-][c-][c-][c-]c1Oc1n[c-]nc([c-]1)Oc1[c-][c-][c-][c-]c1C#N)/C(=O)OC,48866134_207297540,0

COC(=O)C1=C(C)NC(=C([C@@H]1c1cccc(c1)[N+](=O)[O-])C(=O)O[C@H]1CCN(C1)Cc1ccccc1)C.Cl,48866140_48866139,0

O=S1(=O)N[C@H](Cc2[c-][c-][c-][c-][c-]2)Nc2c1[c-]c(c([c-]2)C(F)(F)F)S(=O)(=O)N,48866148_48866147,0

O=S1(=O)N[C@@H](Cc2[c-][c-][c-][c-][c-]2)Nc2c1[c-]c(c([c-]2)C(F)(F)F)S(=O)(=O)N,48866150_48866149,0

[c-]1[c-][c-]c([c-][c-]1)/C=N/N=C/c1[c-][c-][c-][c-][c-]1,48866152_48866151,0

O=C(c1[c-][c-][c-][c-][c-]1)NOCC(=O)O,48866154_48866153,0

CC(CC(c1[c-][c-]c([c-][c-]1)OCCOCC[N+](Cc1[c-][c-][c-][c-][c-]1)(C)C)(C)C)(C)C.[Cl-],48866156_515814,0

O=C1CN(C1)C(c1[c-][c-][c-][c-][c-]1)c1[c-][c-][c-][c-][c-]1,48866158_48866157,0

OC(=O)c1[c-][c-]c2c([c-]1)n[c-]n2,48866160_48866159,0

Cc1c(OCC(F)(F)F)[c-][c-]n2c1c(Sc1nc3c(n1)[c-][c-][c-][c-]3)n1c2nc2c1[c-][c-][c-][c-]2,48866162_48866161,0

CCc1oc2c(c1C(=O)c1[c-]c(I)c(c([c-]1)I)O)[c-][c-][c-][c-]2,48866164_48866163,0

[c-]1[c-]c2[c-]c3c4[c-][c-][c-][c-]c4[c-][c-]c3c3c2c([c-]1)[C-]=[C-]3.[c-]1[c-][c-]c2c([c-]1)[c-]c1c3c2[C-]=[C-]c3[c-]c2c1[c-][c-][c-][c-]2,48866166_48866165,0

O=C1CC(=O)Nc2c(N1)[c-][c-][c-][c-]2,48866168_48866167,0

ClCC(=O)N1[C@@H](Cc2c([C@H]1c1[c-][c-]c3c([c-]1)OCO3)nc1c2[c-][c-][c-][c-]1)C(=O)OC,48866170_207350992,0

deepchem/datasets/tests/test_drop.py

0 → 100644
+63 −0
Original line number Diff line number Diff line 
import os

import shutil

import tempfile

import numpy as np

from deepchem.models.tests import TestAPI

from deepchem.utils.save import load_from_disk

from deepchem.featurizers.featurize import DataFeaturizer

from deepchem.datasets import Dataset

from sklearn.ensemble import RandomForestClassifier

from deepchem.models.sklearn_models import SklearnModel

from deepchem.featurizers.fingerprints import CircularFingerprint



class TestDrop(TestAPI):

  """

  Test how loading of malformed compounds is handled.



  Called TestDrop since these compounds were silently and erroneously dropped.

  """



  def test_drop(self):

    """Test on dataset where RDKit fails on some strings."""

    # Set some global variables up top

    reload = True

    verbosity = "high"

    len_full = 25



    ## This is for good debug (to make sure nasty state isn't being passed around)

    current_dir = os.path.dirname(os.path.realpath(__file__))

    feature_dir = os.path.join(self.base_dir, "features")

    samples_dir = os.path.join(self.base_dir, "samples")

    full_dir = os.path.join(self.base_dir, "full_dataset")

    model_dir = os.path.join(self.base_dir, "model")



    print("About to load emols dataset.")

    dataset_file = os.path.join(

        current_dir, "mini_emols.csv")



    # Featurize emols dataset

    print("About to featurize datasets.")

    featurizers = [CircularFingerprint(size=1024)]

    emols_tasks = ['activity']



    featurizer = DataFeaturizer(tasks=emols_tasks,

                                smiles_field="smiles",

                                compound_featurizers=featurizers,

                                verbosity=verbosity)

    featurized_samples = featurizer.featurize(

        dataset_file, feature_dir,

        samples_dir, reload=reload)

    print("len(featurized_samples)")

    print(len(featurized_samples))



    # Generate datasets

    dataset = Dataset(data_dir=full_dir, samples=featurized_samples, 

                      featurizers=featurizers, tasks=emols_tasks,

                      verbosity=verbosity, reload=reload)

    print("len(dataset)")

    print(len(dataset))



    X, y, w, ids = dataset.to_numpy()

    print("ids.shape, X.shape, y.shape, w.shape")

    print(ids.shape, X.shape, y.shape, w.shape)

    assert len(X) == len(y) == len(w) == len(ids)

deepchem/featurizers/featurize.py

+1 −1
Original line number Diff line number Diff line
@@ -108,7 +108,7 @@ class DataFeaturizer(object): 
    self.log_every_n = log_every_n



  def featurize(self, input_file, feature_dir, samples_dir,

                shard_size=1024, worker_pool=None,

                shard_size=8192, worker_pool=None,

                reload=False):

    """Featurize provided file and write to specified location."""

    # If we are not to reload data, or data has not already been featurized.

deepchem/models/__init__.py

+14 −1
Original line number Diff line number Diff line
@@ -137,16 +137,29 @@ class Model(object): 
    """

    y_preds = []

    batch_size = self.model_params["batch_size"]

    ######################## DEBUG

    print("Model.predict()")

    print("len(dataset)")

    print(len(dataset))

    ######################## DEBUG

    for (X_batch, y_batch, w_batch, ids_batch) in dataset.iterbatches(batch_size):

      y_pred_batch = np.reshape(self.predict_on_batch(X_batch), y_batch.shape)

      y_pred_batch = undo_transforms(y_pred_batch, transformers)

      y_preds.append(y_pred_batch)

      ######################## DEBUG

      print("y_batch.shape, y_pred_batch.shape")

      print(y_batch.shape, y_pred_batch.shape)

      ######################## DEBUG

    y_pred = np.vstack(y_preds)

  

    # The iterbatches does padding with zero-weight examples on the last batch.

    # Remove padded examples.

    n_samples, n_tasks = len(dataset), len(self.tasks)

    y_pred = y_pred[:n_samples]

    ######################## DEBUG

    print("Model.predict()")

    print("y_pred.shape, n_samples, n_tasks")

    print(y_pred.shape, n_samples, n_tasks)

    ######################## DEBUG

    y_pred = np.reshape(y_pred, (n_samples, n_tasks))

    return y_pred

CRA Git | Maintained and supported by SUSTech CRA and CCSE