Commit 59cba5f2 authored by Bharath Ramsundar's avatar Bharath Ramsundar
Browse files

Added new test and BACE ipynb

parent 75284664

datasets/desc_canvas_aug30.csv

0 → 100644
+1523 −0

File added.

Preview size limit exceeded, changes collapsed.

deepchem/featurizers/featurize.py

+6 −6
Original line number Diff line number Diff line
@@ -136,12 +136,12 @@ class DataFeaturizer(object): 
        0, nb_sample, np.ceil(float(nb_sample)/shard_size)+1, dtype=int)

    shard_files = []

    for j in range(len(interval_points)-1):

      log("Sharding and standardizing into shard-%s / %s shards" % (str(j+1), len(interval_points)-1), self.verbose)

      log("Sharding and standardizing into shard-%s / %s shards"

          % (str(j+1), len(interval_points)-1), self.verbose)

      raw_df_shard = raw_df.iloc[range(interval_points[j], interval_points[j+1])]

      

      df = self._standardize_df(raw_df_shard) 

      log("Aggregating User-Specified Features", self.verbose)

      self._add_user_specified_features(df)



      for compound_featurizer in self.compound_featurizers:

        log("Currently feauturizing feature_type: %s"
@@ -198,9 +198,9 @@ class DataFeaturizer(object): 
      df["ligand_pdb"] = ori_df[[self.ligand_pdb_field]]

    if self.ligand_mol2_field is not None:

      df["ligand_mol2"] = ori_df[[self.ligand_mol2_field]]

    self._add_user_specified_features(df, ori_df)

    return df





  def _featurize_complexes(self, df, featurizer, parallel=True,

                           worker_pool=None):

    """Generates circular fingerprints for dataset."""
@@ -254,7 +254,7 @@ class DataFeaturizer(object): 


    df[featurizer.__class__.__name__] = features



  def _add_user_specified_features(self, df):

  def _add_user_specified_features(self, df, ori_df):

    """Merge user specified features. 



      Merge features included in dataset provided by user
@@ -263,7 +263,7 @@ class DataFeaturizer(object): 
    if self.user_specified_features is not None:

      log("Adding user-defined features.", self.verbose)

      features_data = []

      for _, row in df.iterrows():

      for ind, row in ori_df.iterrows():

        # pandas rows are tuples (row_num, row_data)

        feature_list = []

        for feature_name in self.user_specified_features:
@@ -499,7 +499,7 @@ class FeaturizedSamples(object): 
    for ind, row in self.compounds_df.iterrows():

      if row["split"].lower() == "train":

        train_inds.append(ind)

      elif row["split"].lower() == "validation":

      elif row["split"].lower() in ["valid", "validation"]:

        valid_inds.append(ind)

      elif row["split"].lower() == "test":

        test_inds.append(ind)

deepchem/utils/dataset.py

+20 −16
Original line number Diff line number Diff line
@@ -57,17 +57,9 @@ class Dataset(object): 
                   'X_sums', 'X_sum_squares', 'X_n',

                   'y_sums', 'y_sum_squares', 'y_n'))

      self.save_to_disk()

      #save_to_disk(

      #    self.metadata_df, self._get_metadata_filename())

      ## input/output transforms not specified yet, so

      ## self.transforms = (input_transforms, output_transforms) =>

      #self.transforms = ([], [])

      #save_to_disk(

      #    self.transforms, self._get_transforms_filename())

    else:

      if os.path.exists(self._get_metadata_filename()):

        self.metadata_df = load_from_disk(self._get_metadata_filename())

        #self.transforms = load_from_disk(self._get_transforms_filename())

      else:

        raise ValueError("No metadata found.")
@@ -100,12 +92,6 @@ class Dataset(object): 
    metadata_filename = os.path.join(self.data_dir, "metadata.joblib")

    return metadata_filename



  #def _get_transforms_filename(self):

  #  """

  #  Get standard location for stored transforms.

  #  """

  #  return os.path.join(self.data_dir, "transforms.joblib")



  def get_number_shards(self):

    """

    Returns the number of shards for this dataset.
@@ -126,6 +112,16 @@ class Dataset(object): 
      ids = load_from_disk(row['ids'])

      yield (X, y, w, ids)



  def __len__(self):

    """

    Finds number of elements in dataset.

    """

    total = 0

    for _, row in self.metadata_df.iterrows():

      y = load_from_disk(row['y-transformed'])

      total += len(y)

    return total



  def get_label_means(self):

    """Return pandas series of label means."""

    return self.metadata_df["y_means"]
@@ -218,18 +214,26 @@ def _df_to_numpy(df, feature_types, tasks): 
  y = np.reshape(y, (n_samples, n_tasks))

  w = np.ones((n_samples, n_tasks))

  tensors = []

  for _, datapoint in df.iterrows():

  for ind , datapoint in df.iterrows():

    feature_list = []

    for feature_type in feature_types:

      feature_list.append(datapoint[feature_type])

    # TODO(rbharath): Total hack. Fix before merge!!!

    try:

      features = np.squeeze(np.concatenate(feature_list))

      for ind, val in enumerate(features):

        if features[ind] == "":

          features[ind] = 0.

      features = features.astype(float)

    except ValueError:

      y[ind] = ""

      continue

    tensors.append(features)

  x = np.stack(tensors)

  sorted_ids = df["mol_id"]



  # Set missing data to have weight zero

  missing = (y.astype(object) == "")



  y[missing] = 0.

  w[missing] = 0.

deepchem/utils/test/test_api.py

+33 −3
Original line number Diff line number Diff line
@@ -53,7 +53,6 @@ class TestAPI(unittest.TestCase): 
    """Helper method to create model for test."""



    # Fit model



    model.fit(train_dataset)

    model.save(self.model_dir)
@@ -76,6 +75,8 @@ class TestAPI(unittest.TestCase): 
                                  input_transformer_classes,

                                  output_transformer_classes, input_file, tasks, 

                                  protein_pdb_field=None, ligand_pdb_field=None,

                                  user_specified_features=None,

                                  split_field=None,

                                  shard_size=100):

    # Featurize input

    featurizers = compound_featurizers + complex_featurizers
@@ -87,6 +88,8 @@ class TestAPI(unittest.TestCase): 
                                ligand_pdb_field=ligand_pdb_field,

                                compound_featurizers=compound_featurizers,

                                complex_featurizers=complex_featurizers,

                                user_specified_features=user_specified_features,

                                split_field=split_field,

                                verbose=True)



    #Featurizes samples and transforms them into NumPy arrays suitable for ML.
@@ -99,10 +102,13 @@ class TestAPI(unittest.TestCase): 
    train_samples, test_samples = samples.train_test_split(

        splittype, self.train_dir, self.test_dir)



    use_user_specified_features = (user_specified_features is not None)

    train_dataset = Dataset(data_dir=self.train_dir, samples=train_samples, 

                            featurizers=featurizers, tasks=tasks)

                            featurizers=featurizers, tasks=tasks,

                            use_user_specified_features=use_user_specified_features)

    test_dataset = Dataset(data_dir=self.test_dir, samples=test_samples, 

                           featurizers=featurizers, tasks=tasks)

                           featurizers=featurizers, tasks=tasks,

                           use_user_specified_features=use_user_specified_features)



    # Initialize transformers

    input_transformers = []
@@ -144,6 +150,30 @@ class TestAPI(unittest.TestCase): 
    model = SklearnModel(task_types, model_params, model_instance=RandomForestRegressor())

    self._create_model(train_dataset, test_dataset, model, transformers)



  def test_singletask_rf_user_specified_regression_API(self):

    """Test of singletask RF ECFP regression API."""

    splittype = "specified"

    split_field = "split"

    compound_featurizers = []

    complex_featurizers = []

    input_transformers = []

    output_transformers = [NormalizationTransformer]

    model_params = {}

    task_types = {"log-solubility": "regression"}

    input_file = "user_specified_example.csv"

    user_specified_features = ["user-specified1", "user-specified2"]

    train_dataset, test_dataset, _, transformers, = self._featurize_train_test_split(

        splittype, compound_featurizers, 

        complex_featurizers, input_transformers,

        output_transformers, input_file, task_types.keys(),

        user_specified_features=user_specified_features,

        split_field=split_field)

    model_params["data_shape"] = train_dataset.get_data_shape()



    from sklearn.ensemble import RandomForestRegressor

    model = SklearnModel(task_types, model_params, model_instance=RandomForestRegressor())

    self._create_model(train_dataset, test_dataset, model, transformers)



  def test_singletask_rf_ECFP_regression_sharded_API(self):

    """Test of singletask RF ECFP regression API: sharded edition."""

    splittype = "scaffold"

deepchem/utils/test/user_specified_example.csv

0 → 100755
+11 −0
Original line number Diff line number Diff line 
user-specified2,split,user-specified1,Compound ID,log-solubility,smiles

1,train,1,Amigdalin,-0.9740000000000001,OCC3OC(OCC2OC(OC(C#N)c1ccccc1)C(O)C(O)C2O)C(O)C(O)C3O 

0,train,0,Fenfuram,-2.885,Cc1occc1C(=O)Nc2ccccc2

1,train,1,citral,-2.5789999999999997,CC(C)=CCCC(C)=CC(=O)

0,train,0,Picene,-6.617999999999999,c1ccc2c(c1)ccc3c2ccc4c5ccccc5ccc43

1,train,1,Thiophene,-2.2319999999999998,c1ccsc1

0,train,0,benzothiazole,-2.733,c2ccc1scnc1c2 

1,valid,1,"2,2,4,6,6'-PCB",-6.545,Clc1cc(Cl)c(c(Cl)c1)c2c(Cl)cccc2Cl

0,valid,0,Estradiol,-4.138,CC12CCC3C(CCc4cc(O)ccc34)C2CCC1O

1,test,1,Dieldrin,-4.533,ClC4=C(Cl)C5(Cl)C3C1CC(C2OC12)C3C4(Cl)C5(Cl)Cl

0,test,1,Rotenone,-5.246,COc5cc4OCC3Oc2c1CC(Oc1ccc2C(=O)C3c4cc5OC)C(C)=C

CRA Git | Maintained and supported by SUSTech CRA and CCSE