Commit 743a31cd authored by Bharath Ramsundar's avatar Bharath Ramsundar
Browse files

Made featurization use vs_utils API instead of calling script.

parent 8e94c597

deep_chem/scripts/modeler.py

+2 −2
Original line number Diff line number Diff line
@@ -193,8 +193,8 @@ def featurize_input(args): 
      args.smiles_endpoint, args.id_endpoint, out_y_pkl, out_sdf)

  generate_features(df, args.feature_endpoints, args.smiles_endpoint,

                    args.id_endpoint, out_x_pkl)

  generate_fingerprints(args.name, args.out)

  generate_descriptors(args.name, args.out)

  generate_fingerprints(df, args.name, args.out, args.smiles_endpoint, args.id_endpoint)

  generate_descriptors(df, args.name, args.out, args.smiles_endpoint, args.id_endpoint)



def train_model(args):

  """Builds model from featurized data."""

deep_chem/utils/featurize.py

+69 −30
Original line number Diff line number Diff line
@@ -14,6 +14,8 @@ import csv 
from rdkit import Chem

import subprocess

from vs_utils.utils import SmilesGenerator, ScaffoldGenerator

from vs_utils.features.fingerprints import CircularFingerprint

from vs_utils.features.basic import SimpleDescriptors



def generate_directories(name, out, feature_endpoints):

  """Generate directory structure for featurized dataset."""
@@ -57,36 +59,75 @@ def parse_float_input(val): 
    if ">" in val or "<" in val or "-" in val:

      return np.nan



def generate_fingerprints(name, out):

def generate_fingerprints(df, name, out, smiles_endpoint, id_endpoint):

  """Generates circular fingerprints for dataset."""

  dataset_dir = os.path.join(out, name)

  fingerprint_dir = os.path.join(dataset_dir, "fingerprints")

  shards_dir = os.path.join(dataset_dir, "shards")

  sdf = os.path.join(shards_dir, "%s-0.sdf.gz" % name)

  fingerprints = os.path.join(fingerprint_dir,

      "%s-fingerprints.pkl.gz" % name)

  # TODO(rbharath): There's a bit of ugliness here. featurize modifies the

  # smiles strings internally, which I suspect leads to some non-matching

  # smiles strings, hence dropping some compounds. featurize needs to be

  # modified so that it can take in lists of smiles rather than just sdf file.

  # FIXME: Make this directly call the CircularFingerprint featurizer in vs_utils.

  subprocess.call(["python", "-m", "vs_utils.scripts.featurize",

                   "--scaffolds", "--smiles",

                   sdf, fingerprints,

                   "circular", "--size", "1024"])



def generate_descriptors(name, out):



  fingerprint_df = pd.DataFrame([]) 

  fingerprint_df["smiles"] = df[[smiles_endpoint]]

  fingerprint_df["scaffolds"] = df[[smiles_endpoint]].apply(

    functools.partial(generate_scaffold, smiles_endpoint=smiles_endpoint),

    axis=1)

  fingerprint_df["mol_id"] = df[[id_endpoint]]



  mols = []

  for row in df.iterrows():

    # pandas rows are tuples (row_num, row_data)

    smiles = row[1][smiles_endpoint]

    mols.append(Chem.MolFromSmiles(smiles))

  featurizer = CircularFingerprint(size=1024)

  fingerprint_df["features"] = pd.DataFrame([ {"features": feature} for feature in featurizer.featurize(mols)])



  with gzip.open(fingerprints, "wb") as f:

    pickle.dump(fingerprint_df, f, pickle.HIGHEST_PROTOCOL)

  #dataset_dir = os.path.join(out, name)

  #fingerprint_dir = os.path.join(dataset_dir, "fingerprints")

  #shards_dir = os.path.join(dataset_dir, "shards")

  #sdf = os.path.join(shards_dir, "%s-0.sdf.gz" % name)

  #fingerprints = os.path.join(fingerprint_dir,

  #    "%s-fingerprints.pkl.gz" % name)

  ## TODO(rbharath): There's a bit of ugliness here. featurize modifies the

  ## smiles strings internally, which I suspect leads to some non-matching

  ## smiles strings, hence dropping some compounds. featurize needs to be

  ## modified so that it can take in lists of smiles rather than just sdf file.

  ## FIXME: Make this directly call the CircularFingerprint featurizer in vs_utils.

  #subprocess.call(["python", "-m", "vs_utils.scripts.featurize",

  #                 "--scaffolds", "--smiles",

  #                 sdf, fingerprints,

  #                 "circular", "--size", "1024"])



# TODO(rbharath): CODE SMELL! This and generate_fingerprints look almost identical. Factor out into a geneate_standard_featurization function.

def generate_descriptors(df, name, out, smiles_endpoint, id_endpoint):

  """Generates molecular descriptors for dataset."""

  dataset_dir = os.path.join(out, name)

  fingerprint_dir = os.path.join(dataset_dir, "descriptors")

  descriptors_dir = os.path.join(dataset_dir, "descriptors")

  shards_dir = os.path.join(dataset_dir, "shards")

  sdf = os.path.join(shards_dir, "%s-0.sdf.gz" % name)

  descriptors = os.path.join(fingerprint_dir,

  descriptors = os.path.join(descriptors_dir,

      "%s-descriptors.pkl.gz" % name)

  subprocess.call(["python", "-m", "vs_utils.scripts.featurize",

                   "--scaffolds", "--smiles",

                   sdf, descriptors,

                   "descriptors"])



  descriptors_df = pd.DataFrame([]) 

  descriptors_df["smiles"] = df[[smiles_endpoint]]

  descriptors_df["scaffolds"] = df[[smiles_endpoint]].apply(

    functools.partial(generate_scaffold, smiles_endpoint=smiles_endpoint),

    axis=1)

  descriptors_df["mol_id"] = df[[id_endpoint]]

  mols = []

  for row in df.iterrows():

    # pandas rows are tuples (row_num, row_data)

    smiles = row[1][smiles_endpoint]

    mols.append(Chem.MolFromSmiles(smiles))

  featurizer = SimpleDescriptors()

  descriptors_df["features"] = pd.DataFrame([ {"features": feature} for feature in featurizer.featurize(mols)])



  with gzip.open(descriptors, "wb") as f:

    pickle.dump(descriptors_df, f, pickle.HIGHEST_PROTOCOL)

  #subprocess.call(["python", "-m", "vs_utils.scripts.featurize",

  #                 "--scaffolds", "--smiles",

  #                 sdf, descriptors,

  #                 "descriptors"])



def get_rows(input_file, input_type, delimiter):

  """Returns an iterator over all rows in input_file"""
@@ -202,8 +243,13 @@ def generate_features(df, feature_endpoints, smiles_endpoint, id_endpoint, out_p 
    print("No feature endpoint specified by user.")

    return



  features_df = df[[smiles_endpoint]]

  #features_df["features"] = df[[feature_endpoint]]

  features_df = pd.DataFrame([]) 

  features_df["smiles"] = df[[smiles_endpoint]]

  features_df["scaffolds"] = df[[smiles_endpoint]].apply(

    functools.partial(generate_scaffold, smiles_endpoint=smiles_endpoint),

    axis=1)

  features_df["mol_id"] = df[[id_endpoint]]



  features_data = []

  for row in df.iterrows():

    # pandas rows are tuples (row_num, row_data)
@@ -213,13 +259,6 @@ def generate_features(df, feature_endpoints, smiles_endpoint, id_endpoint, out_p 
    features_data.append({"row": np.array(feature_list)})

  features_df["features"] = pd.DataFrame(features_data)



    

  features_df["scaffolds"] = df[[smiles_endpoint]].apply(

    functools.partial(generate_scaffold, smiles_endpoint=smiles_endpoint),

    axis=1)

  features_df["smiles"] = df[[smiles_endpoint]]

  features_df["mol_id"] = df[[id_endpoint]]



  with gzip.open(out_pkl, "wb") as f:

    pickle.dump(features_df, f, pickle.HIGHEST_PROTOCOL)

deep_chem/utils/load.py

+13 −10
Original line number Diff line number Diff line
@@ -73,34 +73,37 @@ def load_molecules(paths, feature_types=["fingerprints"]): 
    List of strings.

  """

  molecules = {}

  print "load_molecules()"

  for dataset_path in paths:

    for feature_type in feature_types:

      print "feature_type: %s" % feature_type

      pickle_dir = os.path.join(dataset_path, feature_type)

      print "pickle_dir: %s" % pickle_dir

      pickle_files = os.listdir(pickle_dir)

      if len(pickle_files) == 0:

        raise ValueError("No Pickle Files found to load molecules")

      for pickle_file in pickle_files:

        print "loading pickle_file: %s" % pickle_file

        with gzip.open(os.path.join(pickle_dir, pickle_file), "rb") as f:

          contents = pickle.load(f)

          print "contents.keys()"

          print contents.keys()

          smiles, features, scaffolds, mol_ids = (

              contents["smiles"], contents["features"],

              contents["scaffolds"], contents["mol_id"])

          splits = contents["split"] if "split" in contents else None

          for mol in range(len(contents["smiles"])):

            if smiles[mol] not in molecules:

              molecules[smiles[mol]] = {"fingerprint": features[mol],

          for mol in range(len(contents["mol_id"])):

            if mol_ids[mol] not in molecules:

              molecules[mol_ids[mol]] = {"fingerprint": features[mol],

                                         "scaffold": scaffolds[mol],

                                         "mol_id": mol_ids[mol],

                                         "feature_types": [feature_type]}

            if splits is not None:

              molecules[smiles[mol]]["split"] = splits[mol]

            # TODO(rbharath): Our processing pipeline sometimes makes different

            # molecules look the same (due to bugs in how we hydrogenate for

            # example). Fix these bugs in our processing pipeline.

            elif feature_type not in molecules[smiles[mol]]["feature_types"]:

              entry = molecules[smiles[mol]]

              molecules[mol_ids[mol]]["split"] = splits[mol]

            elif feature_type not in molecules[mol_ids[mol]]["feature_types"]:

              entry = molecules[mol_ids[mol]]

              entry["fingerprint"] = np.append(

                  molecules[smiles[mol]]["fingerprint"], features[mol])

                  molecules[mol_ids[mol]]["fingerprint"], features[mol])

              entry["feature_types"].append(feature_type)

  return molecules

deep_chem/utils/preprocess.py

+3 −0
Original line number Diff line number Diff line
@@ -173,6 +173,9 @@ def dataset_to_numpy(dataset, feature_endpoint="fingerprint", 
        W[index][t_ind] = 0

      else:

        y[index][t_ind] = labels[target]

  print "DATASET_TO_NUMPY"

  print "np.shape(X)"

  print np.shape(X)

  if weight_positives:

    W = balance_positives(y, W)

  return (X, y, W)

CRA Git | Maintained and supported by SUSTech CRA and CCSE