Merge pull request #1 from deepchem/master

language: python

python:

- '2.7'

sudo: required

install:

- sudo apt-get update

- wget http://repo.continuum.io/archive/Anaconda2-2.4.1-Linux-x86_64.sh -O anaconda.sh;

- conda install joblib

- conda install -c omnia theano

- conda install -c omnia keras

- conda install seaborn

- conda install six

- python setup.py install

script:

- nosetests -v deepchem

after_success:

- source devtools/travis-ci/after_sucess.sh

# AWS access_key and secret key secured through travis secure var api.

env:

  global:

  - secure: FojR+Zrw/XLeDWHaZnwqkBt7DGwOJ6N9X0HJIkkzGsTxQy405qY9bfKMgzivY4UAppqWH2R9kcY/3K2lnFOU9Z2dSf9ZSLVmXjQQK+YhiFiPVQG1bQ8XEhQ51927AZn2wGqzPSFlOI8M4ek2V4a6d72Wg6SOooVbI+A9PtrdXNJlEfonY0QfSakhg+IFPBsN84khbGaHlTRCXvozRTSm3Ubo4jidN54WbO7Ll18qJ183Fgh8SVxYfodDAusZaWU2e/q9lCk6nkJ9bDC9anPbuTXpSlIkH7x7WcrMbLCrvvHJq26YOCjReCDea1/N8ECsarXWfvPjz1Cv59QSghoLxFeRRNCDHWaWTccKgWRAFd60+vSgd8CcoMFsMY0XedO2mSQztLAb2pS2TCoCPMD/5Zip5pkXOxJB5CwPqSuJ41FBPYthzyhG1MPbaQnPDZjtx5TDaVdN5w+Nfc40i9+0rFNisEyAdi2dTcr6bWZ7UlWj7LBDJGVjYPC8lKsaip6dyov3KeLveod0IbY5zCVnqF2rwuID5n/6ejGgytv1caZWa9TvEjzuSJoo8d2uXwyTzQLntardt2ZvtA8ZmwChyaq0pNtUgxQE4wn8hZG19+lAvIc95ZDJuSV7xwuo5XsMDffa3jrjyWhByveyfbHs1znAmAl4w6NJ32ylWr0/+Do=

import pandas as pd

import numpy as np

import csv

import numbers

from rdkit import Chem

from deepchem.featurizers.fingerprints import CircularFingerprint

from deepchem.featurizers.basic import RDKitDescriptors

def _process_field(val):

  """Parse data in a field."""

  if isinstance(val, float) or isinstance(val, np.ndarray):

  if (isinstance(val, numbers.Number) or isinstance(val, np.ndarray)):

    return val

  elif isinstance(val, list):

    return [_process_field(elt) for elt in val]

                                                input_type=input_type)

    #processed_rows = raw_df.apply(process_raw_sample_helper_partial, axis=1)

    raw_df = raw_df.apply(process_raw_sample_helper_partial, axis=1, reduce=False)

    #raw_df = pd.DataFrame.from_records(processed_rows)

    nb_sample = raw_df.shape[0]

    interval_points = np.linspace(

        0, nb_sample, np.ceil(float(nb_sample)/shard_size)+1, dtype=int)

      save_to_disk(compounds_df, self._get_compounds_filename())

    _check_validity(compounds_df)

    self.compounds_df = compounds_df

    self.num_samples = len(compounds_df)

    if os.path.exists(self._get_dataset_paths_filename()):

      if dataset_files is not None:

    _check_validity(df)

    save_to_disk(df, self._get_compounds_filename())

    self.compounds_df = df

    self.num_samples = len(df)

  def __len__(self):

    """Returns size of internal dataset."""

    return self.num_samples

  # TODO(rbharath): Might this be inefficient?

  def itersamples(self):

    """Iterates over samples in this object."""

    compound_ids = set(list(self.compounds_df["mol_id"]))

    for df_file in self.dataset_files:

      df = load_from_disk(df_file)

      visible_inds = []

      for ind, row in df.iterrows():

        if row["mol_id"] in compound_ids:

          visible_inds.append(ind)

      for visible_ind in visible_inds:

        yield df.loc[visible_ind]

  # TODO(rbharath): Might this be inefficient?

  def iterdataframes(self):

    """

    Provides an iterator over samples.

    Provides a bulk iterator over data.

    Each sample from the iterator is a dataframe of samples.

    """

          visible_inds.append(ind)

      yield df.loc[visible_inds]

  def train_test_split(self, splittype, train_dir, test_dir, seed=None,

                       frac_train=.8):

  def train_valid_test_split(self, splittype, train_dir=None,

                             valid_dir=None, test_dir=None, frac_train=.8,

                             frac_valid=.1, frac_test=.1, seed=None):

    """

    Splits self into train/test sets and returns two FeaturizedDatsets

    Splits self into train/validation/test sets.

    Returns FeaturizedDataset objects.

    """

    if splittype == "random":

      train_inds, test_inds = self._train_test_random_split(seed=seed, frac_train=frac_train)

      train_inds, valid_inds, test_inds = self._random_split(

          seed=seed, frac_train=frac_train, frac_test=frac_test,

          frac_valid=frac_valid)

    elif splittype == "scaffold":

      train_inds, test_inds = self._train_test_scaffold_split(frac_train=frac_train)

      train_inds, valid_inds, test_inds = self._scaffold_split(

          frac_train=frac_train, frac_test=frac_test,

          frac_valid=frac_valid)

    elif splittype == "specified":

      train_inds, test_inds = self._train_test_specified_split()

      train_inds, valid_inds, test_inds = self._specified_split()

    else:

      raise ValueError("improper splittype.")

    train_samples, valid_samples, test_samples = None, None, None

    if train_dir is not None:

      train_samples = FeaturizedSamples(samples_dir=train_dir, 

                                        dataset_files=self.dataset_files,

                                        featurizers=self.featurizers)

      train_samples._set_compound_df(self.compounds_df.iloc[train_inds])

    if test_dir is not None:

      test_samples = FeaturizedSamples(samples_dir=test_dir, 

                                       dataset_files=self.dataset_files,

                                       featurizers=self.featurizers)

      test_samples._set_compound_df(self.compounds_df.iloc[test_inds])

    if valid_dir is not None:

      valid_samples = FeaturizedSamples(samples_dir=valid_dir, 

                                       dataset_files=self.dataset_files,

                                       featurizers=self.featurizers)

      valid_samples._set_compound_df(self.compounds_df.iloc[valid_inds])

    return train_samples, valid_samples, test_samples

  def train_test_split(self, splittype, train_dir, test_dir, seed=None,

                       frac_train=.8):

    """

    Splits self into train/test sets.

    Returns FeaturizedDataset objects.

    """

    train_samples, _, test_samples = self.train_valid_test_split(

        splittype, train_dir, valid_dir=None, test_dir=test_dir,

        frac_train=frac_train, frac_test=1-frac_train, frac_valid=0.)

    return train_samples, test_samples

  def _train_test_random_split(self, seed=None, frac_train=.8):

  def _random_split(self, seed=None, frac_train=.8, frac_valid=.1,

                    frac_test=.1):

    """

    Splits internal compounds randomly into train/test.

    Splits internal compounds randomly into train/validation/test.

    """

    np.testing.assert_almost_equal(frac_train + frac_valid + frac_test, 1.)

    np.random.seed(seed)

    train_cutoff = frac_train * len(self.compounds_df)

    valid_cutoff = (frac_train+frac_valid) * len(self.compounds_df)

    shuffled = np.random.permutation(range(len(self.compounds_df)))

    return shuffled[:train_cutoff], shuffled[train_cutoff:]

    return (shuffled[:train_cutoff], shuffled[train_cutoff:valid_cutoff],

            shuffled[valid_cutoff:])

  def _train_test_scaffold_split(self, frac_train=.8):

  def _scaffold_split(self, frac_train=.8, frac_valid=.1, frac_test=.1):

    """

    Splits internal compounds into train/test by scaffold.

    Splits internal compounds into train/validation/test by scaffold.

    """

    np.testing.assert_almost_equal(frac_train + frac_valid + frac_test, 1.)

    scaffolds = {}

    for ind, row in self.compounds_df.iterrows():

      scaffold = generate_scaffold(row["smiles"])

    scaffold_sets = [scaffold_set for (scaffold, scaffold_set) in

                     sorted(scaffolds.items(), key=lambda x: -len(x[1]))]

    train_cutoff = frac_train * len(self.compounds_df)

    train_inds, test_inds = [], []

    valid_cutoff = (frac_train+frac_valid) * len(self.compounds_df)

    train_inds, valid_inds, test_inds = [], [], []

    for scaffold_set in scaffold_sets:

      if len(train_inds) + len(scaffold_set) > train_cutoff:

        if len(train_inds) + len(valid_inds) + len(scaffold_set) > valid_cutoff:

          test_inds += scaffold_set

        else:

          valid_inds += scaffold_set

      else:

        train_inds += scaffold_set

    return train_inds, test_inds

    return train_inds, valid_inds, test_inds

  def _train_test_specified_split(self):

  def _specified_split(self):

    """

    Splits internal compounds into train/test by user-specification.

    Splits internal compounds into train/validation/test by user-specification.

    """

    train_inds, test_inds = [], []

    train_inds, valid_inds, test_inds = [], [], []

    for ind, row in self.compounds_df.iterrows():

      if row["split"].lower() == "train":

        train_inds.append(ind)

      elif row["split"].lower() == "validation":

        valid_inds.append(ind)

      elif row["split"].lower() == "test":

        test_inds.append(ind)

      else:

        raise ValueError("Missing required split information.")

    return train_inds, test_inds

    return train_inds, valid_inds, test_inds

"""

The following code implements a featurizer based on NNScore 2.0.1

Protein-ligand noncovalent chemistry descriptors

"""

from __future__ import print_function

from __future__ import division

"""

Tests for FeaturizedSamples class

"""

from __future__ import print_function

from __future__ import division

from __future__ import unicode_literals

__author__ = "Bharath Ramsundar"

__copyright__ = "Copyright 2016, Stanford University"

__license__ = "LGPL"

import os

import unittest

import tempfile

import shutil

from deepchem.featurizers.featurize import DataFeaturizer

from deepchem.featurizers.fingerprints import CircularFingerprint

class TestFeaturizedSamples(unittest.TestCase):

  """

  Test Featurized Samples class.

  """

  def setUp(self):

    self.current_dir = os.path.dirname(os.path.abspath(__file__))

    self.smiles_field = "smiles"

    self.feature_dir = tempfile.mkdtemp()

    self.samples_dir = tempfile.mkdtemp()

    self.train_dir = tempfile.mkdtemp()

    self.valid_dir = tempfile.mkdtemp()

    self.test_dir = tempfile.mkdtemp()

  def _featurize_train_valid_test_split(self, splittype, input_file, tasks,

                                        frac_train, frac_valid, frac_test):

    # Featurize input

    compound_featurizers = [CircularFingerprint(size=1024)]

    complex_featurizers = []

    featurizers = compound_featurizers + complex_featurizers

    input_file = os.path.join(self.current_dir, input_file)

    featurizer = DataFeaturizer(tasks=tasks,

                                smiles_field=self.smiles_field,

                                compound_featurizers=compound_featurizers,

                                complex_featurizers=complex_featurizers,

                                verbose=True)

    #Featurizes samples and transforms them into NumPy arrays suitable for ML.

    #returns an instance of class FeaturizedSamples()

    samples = featurizer.featurize(input_file, self.feature_dir, self.samples_dir)

    # Splits featurized samples into train/test

    if frac_valid > 0:

      train_samples, valid_samples, test_samples = samples.train_valid_test_split(

          splittype, train_dir=self.train_dir, valid_dir=self.valid_dir,

          test_dir=self.test_dir, frac_train=frac_train,

          frac_valid=frac_valid, frac_test=frac_test)

      return train_samples, valid_samples, test_samples

    else:

      train_samples, test_samples = samples.train_test_split(

          splittype, train_dir=self.train_dir, test_dir=self.test_dir,

          frac_train=frac_train)

      return train_samples, test_samples

  def scaffold_test_train_valid_test_split(self):

    """Test of singletask RF ECFP regression API."""

    splittype = "scaffold"

    input_transforms = []

    output_transforms = ["normalize"]

    model_params = {}

    task_types = {"log-solubility": "regression"}

    input_file = "../../utils/test/example.csv"

    train_samples, valid_samples, test_samples = (

        self._featurize_train_valid_test_split(

            splittype, input_file, task_types.keys(), frac_train=.8,

            frac_valid=.1, frac_test=.1))

    assert len(train_samples) == 8

    assert len(valid_samples) == 1

    assert len(test_samples) == 1

  def scaffold_test_train_test_split(self):

    """Test of singletask RF ECFP regression API."""

    splittype = "scaffold"

    input_transforms = []

    output_transforms = ["normalize"]

    model_params = {}

    task_types = {"log-solubility": "regression"}

    input_file = "../../utils/test/example.csv"

    train_samples, test_samples = (

        self._featurize_train_valid_test_split(

            splittype, input_file, task_types.keys(), frac_train=.8,

            frac_valid=0, frac_test=.2))

    assert len(train_samples) == 8

    assert len(test_samples) == 2

  def random_test_train_valid_test_split(self):

    """Test of singletask RF ECFP regression API."""

    splittype = "random"

    input_transforms = []

    output_transforms = ["normalize"]

    model_params = {}

    task_types = {"log-solubility": "regression"}

    input_file = "../../utils/test/example.csv"

    train_samples, valid_samples, test_samples = (

        self._featurize_train_valid_test_split(

            splittype, input_file, task_types.keys(), frac_train=.8,

            frac_valid=.1, frac_test=.1))

    assert len(train_samples) == 8

    assert len(valid_samples) == 1

    assert len(test_samples) == 1

  def random_test_train_test_split(self):

    """Test of singletask RF ECFP regression API."""

    splittype = "random"

    input_transforms = []

    output_transforms = ["normalize"]

    model_params = {}

    task_types = {"log-solubility": "regression"}

    input_file = "../../utils/test/example.csv"

    train_samples, test_samples = (

        self._featurize_train_valid_test_split(

            splittype, input_file, task_types.keys(), frac_train=.8,

            frac_valid=0, frac_test=.2))

    assert len(train_samples) == 8

    assert len(test_samples) == 2