Merge branch 'master' into cut-131

  valid_inds = np.array(

      [1 if elt.size > 0 else 0 for elt in features], dtype=bool)

  features = [elt for (is_valid, elt) in zip(valid_inds, features) if is_valid]

  return np.squeeze(np.array(features)), valid_inds

  return np.squeeze(np.array(features), axis=1), valid_inds

def featurize_smiles_np(arr, featurizer, log_every_N=1000, verbose=True):

      [1 if elt.size > 0 else 0 for elt in features], dtype=bool)

  features = [elt for (is_valid, elt) in zip(valid_inds, features) if is_valid]

  features = np.squeeze(np.array(features))

  return features.reshape(

      -1,)

  return features.reshape(-1,)

def get_user_specified_features(df, featurizer, verbose=True):

from __future__ import print_function

from __future__ import division

from __future__ import unicode_literals

import json

import os

import math

import numpy as np

import pandas as pd

import random

from deepchem.utils.save import save_to_disk

from deepchem.utils.save import save_to_disk, save_metadata

from deepchem.utils.save import load_from_disk

from deepchem.utils.save import log

from pandas import read_hdf

import tempfile

import time

import shutil

    self.verbose = verbose

    log("Loading dataset from disk.", self.verbose)

    if os.path.exists(self._get_metadata_filename()):

      (self.tasks,

       self.metadata_df) = load_from_disk(self._get_metadata_filename())

    else:

      raise ValueError("No metadata found on disk.")

    self.tasks, self.metadata_df = self.load_metadata()

  @staticmethod

  def create_dataset(shard_generator, data_dir=None, tasks=[], verbose=True):

          DiskDataset.write_data_to_disk(data_dir, basename, tasks, X, y, w,

                                         ids))

    metadata_df = DiskDataset._construct_metadata(metadata_rows)

    metadata_filename = os.path.join(data_dir, "metadata.joblib")

    save_to_disk((tasks, metadata_df), metadata_filename)

    save_metadata(tasks, metadata_df, data_dir)

    time2 = time.time()

    log("TIMING: dataset construction took %0.3f s" % (time2 - time1), verbose)

    return DiskDataset(data_dir, verbose=verbose)

  def load_metadata(self):

    try:

      tasks_filename, metadata_filename = self._get_metadata_filename()

      with open(tasks_filename) as fin:

        tasks = json.load(fin)

      metadata_df = pd.read_csv(metadata_filename, compression='gzip')

      metadata_df = metadata_df.where((pd.notnull(metadata_df)), None)

      return tasks, metadata_df

    except Exception as e:

      pass

    # Load obsolete format -> save in new format

    metadata_filename = os.path.join(self.data_dir, "metadata.joblib")

    if os.path.exists(metadata_filename):

      tasks, metadata_df = load_from_disk(metadata_filename)

      del metadata_df['task_names']

      del metadata_df['basename']

      save_metadata(tasks, metadata_df, self.data_dir)

      return tasks, metadata_df

    raise ValueError("No Metadata Found On Disk")

  @staticmethod

  def _construct_metadata(metadata_entries):

    """Construct a dataframe containing metadata.

    metadata_entries should have elements returned by write_data_to_disk

    above.

    """

    columns = ('basename', 'task_names', 'ids', 'X', 'y', 'w')

    columns = ('ids', 'X', 'y', 'w')

    metadata_df = pd.DataFrame(metadata_entries, columns=columns)

    return metadata_df

      out_ids = None

    # note that this corresponds to the _construct_metadata column order

    return [basename, tasks, out_ids, out_X, out_y, out_w]

    return [out_ids, out_X, out_y, out_w]

  def save_to_disk(self):

    """Save dataset to disk."""

    save_to_disk((self.tasks, self.metadata_df), self._get_metadata_filename())

    save_metadata(self.tasks, self.metadata_df, self.data_dir)

  def move(self, new_data_dir):

    """Moves dataset to new directory."""

    """

    Get standard location for metadata file.

    """

    metadata_filename = os.path.join(self.data_dir, "metadata.joblib")

    return metadata_filename

    metadata_filename = os.path.join(self.data_dir, "metadata.csv.gzip")

    tasks_filename = os.path.join(self.data_dir, "tasks.json")

    return tasks_filename, metadata_filename

  def get_number_shards(self):

    """

    n_rows = len(self.metadata_df.index)

    for i in range(n_rows):

      row = self.metadata_df.iloc[i]

      basename = row["basename"]

      X, y, w, ids = self.get_shard(i)

      n = X.shape[0]

      permutation = np.random.permutation(n)

      X, y, w, ids = (X[permutation], y[permutation], w[permutation],

                      ids[permutation])

      DiskDataset.write_data_to_disk(self.data_dir, basename, tasks, X, y, w,

                                     ids)

      DiskDataset.write_data_to_disk(self.data_dir, "", tasks, X, y, w, ids)

  def shuffle_shards(self):

    """Shuffles the order of the shards for this dataset."""

import os

from unittest import TestCase

from io import StringIO

import tempfile

import shutil

import deepchem as dc

class TestCSVLoader(TestCase):

  def test_load_singleton_csv(self):

    fin = tempfile.NamedTemporaryFile(mode='w', delete=False)

    fin.write("smiles,endpoint\nc1ccccc1,1")

    fin.close()

    print(fin.name)

    featurizer = dc.feat.CircularFingerprint(size=1024)

    tasks = ["endpoint"]

    loader = dc.data.CSVLoader(

        tasks=tasks, smiles_field="smiles", featurizer=featurizer)

    X = loader.featurize(fin.name)

    self.assertEqual(1, len(X))

    os.remove(fin.name)

    if feat == "grid":

      self.featurizer = RdkitGridFeaturizer(

          voxel_width=16.0,

          feature_types="voxel_combined",

          # TODO(rbharath, enf): Figure out why pi_stack is slow and cation_pi

          # causes segfaults.

          #voxel_feature_types=["ecfp", "splif", "hbond", "pi_stack", "cation_pi",

          #"salt_bridge"], ecfp_power=9, splif_power=9,

          voxel_feature_types=["ecfp", "splif", "hbond", "salt_bridge"],

          # TODO: add pi_stack and cation_pi to feature_types (it's not trivial

          # because they require sanitized molecules)

          # feature_types=["ecfp", "splif", "hbond", "pi_stack", "cation_pi",

          # "salt_bridge"],

          feature_types=["ecfp", "splif", "hbond", "salt_bridge"],

          ecfp_power=9,

          splif_power=9,

          parallel=True,

          flatten=True)

    else:

      raise ValueError("feat not defined.")