Merge branch 'master' into material_featurizer_renames

from deepchem.data.data_loader import SDFLoader

from deepchem.data.data_loader import FASTALoader

from deepchem.data.data_loader import ImageLoader

from deepchem.data.data_loader import InMemoryLoader

import sys

import logging

import warnings

from typing import List, Optional, Dict, Tuple

from typing import List, Optional, Dict, Tuple, Any, Sequence, Union

from deepchem.utils.typing import OneOrMany

from deepchem.utils.save import load_csv_files, load_json_files

from deepchem.utils.save import load_sdf_files

from deepchem.utils.genomics import encode_fasta_sequence

from deepchem.feat import UserDefinedFeaturizer, Featurizer

from deepchem.data import DiskDataset, NumpyDataset, ImageDataset

from deepchem.data import Dataset, DiskDataset, NumpyDataset, ImageDataset

import zipfile

logger = logging.getLogger(__name__)

    The name of a column in `df` that holds SMILES strings

  log_every_n: int, optional (default 1000)

    Emit a logging statement every `log_every_n` rows.

  Note

  ----

  This function requires RDKit to be installed

  """

  sample_elems = df[field].tolist()

    self.featurizer = featurizer

    self.log_every_n = log_every_n

  def featurize(self, input_files, data_dir=None, shard_size=8192):

  def featurize(self,

                inputs: Sequence[Any],

                data_dir: Optional[str] = None,

                shard_size: Optional[int] = 8192) -> Dataset:

    """Featurize provided files and write to specified location.

    DEPRECATED: This method is now a wrapper for `create_dataset()`

    Parameters

    ----------

    input_files: list

      List of input filenames.

    inputs: list

      List of inputs to process. Entries can be filenames or arbitrary objects.

    data_dir: str, optional

      Directory to store featurized dataset.

    shard_size: int, optional

    Returns

    -------

    A `Dataset` object containing a featurized representation of data

    from `input_files`.

    from `input`.

    """

    warnings.warn(

        "featurize() is deprecated and has been renamed to create_dataset(). featurize() will be removed in DeepChem 3.0",

        FutureWarning)

    return self.create_dataset(input_files, data_dir, shard_size)

    return self.create_dataset(inputs, data_dir, shard_size)

  def create_dataset(self, input_files, data_dir=None, shard_size=8192):

  def create_dataset(self,

                     inputs: Sequence[Any],

                     data_dir: Optional[str] = None,

                     shard_size: Optional[int] = 8192) -> Dataset:

    """Creates and returns a `Dataset` object by featurizing provided files.

    Reads in `input_files` and uses `self.featurizer` to featurize the

    data in these input files.  For large files, automatically shards

    Reads in `inputs` and uses `self.featurizer` to featurize the

    data in these inputs.  For large files, automatically shards

    into smaller chunks of `shard_size` datapoints for convenience.

    Returns a `Dataset` object that contains the featurized dataset.

    Parameters

    ----------

    input_files: list

      List of input filenames.

    inputs: list

      List of inputs to process. Entries can be filenames or arbitrary objects.

    data_dir: str, optional

      Directory to store featurized dataset.

    shard_size: int, optional

    Returns

    -------

    A `Dataset` object containing a featurized representation of data

    from `input_files`.

    from `inputs`.

    """

    logger.info("Loading raw samples now.")

    logger.info("shard_size: %d" % shard_size)

    logger.info("shard_size: %s" % str(shard_size))

    if not isinstance(input_files, list):

      input_files = [input_files]

    if not isinstance(inputs, list):

      inputs = [inputs]

    def shard_generator():

      for shard_num, shard in enumerate(

          self._get_shards(input_files, shard_size)):

      for shard_num, shard in enumerate(self._get_shards(inputs, shard_size)):

        time1 = time.time()

        X, valid_inds = self._featurize_shard(shard)

        ids = shard[self.id_field].values

    return DiskDataset.create_dataset(shard_generator(), data_dir, self.tasks)

  def _get_shards(self, input_files, shard_size):

  def _get_shards(self, inputs, shard_size):

    """Stub for children classes.

    Should implement a generator that walks over the source data in

    `input_files` and returns a "shard" at a time. Here a shard is a

    `inputs` and returns a "shard" at a time. Here a shard is a

    chunk of input data that can reasonably be handled in memory. For

    example, this may be a set of rows from a CSV file or a set of

    molecules from a SDF file. To re-use the

    Parameters

    ----------

    input_files: list

      List of input filenames.

    inputs: list

      List of inputs to process. Entries can be filenames or arbitrary objects.

    shard_size: int, optional

      Number of examples stored in each shard.

    """

    self.log_every_n = log_every_n

  def _get_shards(self, input_files, shard_size):

    """Defines a generator which returns data for each shard"""

    """Defines a generator which returns data for each shard

    Parameters

    ----------

    input_files: list[str]

      List of filenames to process

    shard_size: int

      The size of a shard of data to process at a time.

    """

    return load_csv_files(input_files, shard_size)

  def _featurize_shard(self, shard):

  """

  def __init__(self,

               tasks: List[str],

               tasks: OneOrMany[str],

               feature_field: str,

               label_field: str = None,

               weight_field: str = None,

    self.log_every_n = log_every_n

  def create_dataset(self,

                     input_files: List[str],

                     input_files: OneOrMany[str],

                     data_dir: Optional[str] = None,

                     shard_size: Optional[int] = 8192) -> DiskDataset:

    """Creates a `Dataset` from input JSON files.

    Parameters

    ----------

    input_files: List[str]

    input_files: OneOrMany[str]

      List of JSON filenames.

    data_dir: Optional[str], default None

      Name of directory where featurized data is stored.

      from `input_files`.

    """

    if not isinstance(input_files, list):

      try:

        if isinstance(input_files, str):

          input_files = [input_files]

        else:

          input_files = list(input_files)

      except TypeError:

        raise ValueError(

            "input_files is of an unrecognized form. Must be one filename or a list of filenames."

        )

    def shard_generator():

      """Yield X, y, w, and ids for shards."""

    """Initialize loader."""

    pass

  def create_dataset(self, input_files, data_dir=None, shard_size=None):

  def create_dataset(self,

                     input_files: OneOrMany[str],

                     data_dir: Optional[str] = None,

                     shard_size: Optional[int] = None) -> DiskDataset:

    """Creates a `Dataset` from input FASTA files.

    At present, FASTA support is limited and only allows for one-hot

    A `Dataset` object containing a featurized representation of data

    from `input_files`.

    """

    if not isinstance(input_files, list):

    if isinstance(input_files, str):

      input_files = [input_files]

    def shard_generator():

  traverse subdirectories which contain images.

  """

  def __init__(self, tasks=None):

  def __init__(self, tasks: OneOrMany[str] = None):

    """Initialize image loader.

    At present, custom image featurizers aren't supported by this

    self.tasks = tasks

  def create_dataset(self,

                     input_files,

                     labels=None,

                     weights=None,

                     in_memory=False):

                     inputs: Union[OneOrMany[str], Tuple[Any]],

                     data_dir: Optional[str] = None,

                     shard_size: Optional[int] = 8192,

                     in_memory: bool = False) -> Dataset:

    """Creates and returns a `Dataset` object by featurizing provided image files and labels/weights.

    Parameters

    ----------

    input_files: list

      Each file in this list should either be of a supported

      image format (.png, .tif only for now) or of a compressed

      folder of image files (only .zip for now).

    labels: optional

      If provided, a numpy ndarray of image labels

    weights: optional

      If provided, a numpy ndarray of image weights

    inputs: `Union[OneOrMany[str], Tuple[Any]]`

      The inputs provided should be one of the following

      - filename

      - list of filenames

      - Tuple (list of filenames, labels)

      - Tuple (list of filenames, labels, weights)

      Each file in a given list of filenames should either be of a supported

      image format (.png, .tif only for now) or of a compressed folder of

      image files (only .zip for now). If `labels` or `weights` are provided,

      they must correspond to the sorted order of all filenames provided, with

      one label/weight per file.

    data_dir: str, optional

      Directory to store featurized dataset.

    in_memory: bool

      If true, return in-memory NumpyDataset. Else return ImageDataset.

    -------

    A `Dataset` object containing a featurized representation of data

    from `input_files`, `labels`, and `weights`.

    """

    if not isinstance(input_files, list):

    labels, weights = None, None

    if isinstance(inputs, tuple):

      if len(inputs) == 1:

        input_files = inputs[0]

        if isinstance(inputs, str):

          input_files = [inputs]

      elif len(inputs) == 2:

        input_files, labels = inputs

      elif len(inputs) == 3:

        input_files, labels, weights = inputs

      else:

        raise ValueError("Input must be a tuple of length 1, 2, or 3")

    else:

      input_files = inputs

    if isinstance(input_files, str):

      input_files = [input_files]

    image_files = []

          raise ValueError("Unsupported file format")

      input_files = remainder

    # Sort image files

    image_files = sorted(image_files)

    if in_memory:

      if data_dir is None:

        return NumpyDataset(

            self.load_img(image_files), y=labels, w=weights, ids=image_files)

      else:

        dataset = DiskDataset.from_numpy(

            self.load_img(image_files),

            y=labels,

            w=weights,

            ids=image_files,

            tasks=self.tasks,

            data_dir=data_dir)

        if shard_size is not None:

          dataset.reshard(shard_size)

        return dataset

    else:

      return ImageDataset(image_files, y=labels, w=weights, ids=image_files)

  @staticmethod

  def load_img(image_files):

  def load_img(image_files) -> np.ndarray:

    """Loads a set of images from disk.

    Parameters

    ----------

    image_files: list[str]

      List of image filenames to load

    Returns

    -------

    np.ndarray that contains loaded images. Of shape `(N,...)`.

    Note

    ----

    This method requires PIL to be installed.

    """

    from PIL import Image

    images = []

    for image_file in image_files:

      else:

        raise ValueError("Unsupported image filetype for %s" % image_file)

    return np.array(images)

class InMemoryLoader(DataLoader):

  """Facilitate Featurization of In-memory objects.

  When featurizing a dataset, it's often the case that the initial set of

  data (pre-featurization) fits handily within memory. (For example, perhaps

  it fits within a column of a pandas DataFrame.) In this case, it would be

  convenient to directly be able to featurize this column of data. However,

  the process of featurization often generates large arrays which quickly eat

  up available memory. This class provides convenient capabilities to process

  such in-memory data by checkpointing generated features periodically to

  disk.

  Example

  -------

  Here's an example with only datapoints and no labels or weights.

  >>> import deepchem as dc

  >>> smiles = ["C", "CC", "CCC", "CCCC"]

  >>> featurizer = dc.feat.CircularFingerprint()

  >>> loader = dc.data.InMemoryLoader(tasks=["task1"], featurizer=featurizer)

  >>> dataset = loader.create_dataset(smiles, shard_size=2)

  >>> len(dataset)

  4

  Here's an example with both datapoints and labels

  >>> import deepchem as dc

  >>> smiles = ["C", "CC", "CCC", "CCCC"]

  >>> labels = [1, 0, 1, 0]

  >>> featurizer = dc.feat.CircularFingerprint()

  >>> loader = dc.data.InMemoryLoader(tasks=["task1"], featurizer=featurizer)

  >>> dataset = loader.create_dataset(zip(smiles, labels), shard_size=2)

  >>> len(dataset)

  4

  Here's an example with datapoints, labels, weights and ids all provided.

  >>> import deepchem as dc

  >>> smiles = ["C", "CC", "CCC", "CCCC"]

  >>> labels = [1, 0, 1, 0]

  >>> weights = [1.5, 0, 1.5, 0]

  >>> ids = ["C", "CC", "CCC", "CCCC"]

  >>> featurizer = dc.feat.CircularFingerprint()

  >>> loader = dc.data.InMemoryLoader(tasks=["task1"], featurizer=featurizer)

  >>> dataset = loader.create_dataset(zip(smiles, labels, weights, ids), shard_size=2)

  >>> len(dataset)

  4

  """

  def create_dataset(self,

                     inputs: Sequence[Any],

                     data_dir: Optional[str] = None,

                     shard_size: Optional[int] = 8192) -> DiskDataset:

    """Creates and returns a `Dataset` object by featurizing provided files.

    Reads in `inputs` and uses `self.featurizer` to featurize the

    data in these input files.  For large files, automatically shards

    into smaller chunks of `shard_size` datapoints for convenience.

    Returns a `Dataset` object that contains the featurized dataset.

    This implementation assumes that the helper methods `_get_shards`

    and `_featurize_shard` are implemented and that each shard

    returned by `_get_shards` is a pandas dataframe.  You may choose

    to reuse or override this method in your subclass implementations.

    Parameters

    ----------

    inputs: Sequence[Any]

      List of inputs to process. Entries can be arbitrary objects so long as

      they are understood by `self.featurizer`

    data_dir: str, optional

      Directory to store featurized dataset.

    shard_size: int, optional

      Number of examples stored in each shard.

    Returns

    -------

    A `Dataset` object containing a featurized representation of data

    from `inputs`.

    """

    logger.info("Loading raw samples now.")

    logger.info("shard_size: %s" % str(shard_size))

    if not isinstance(inputs, list):

      try:

        inputs = list(inputs)

      except TypeError:

        inputs = [inputs]

    def shard_generator():

      global_index = 0

      for shard_num, shard in enumerate(self._get_shards(inputs, shard_size)):

        time1 = time.time()

        X, y, w, ids = self._featurize_shard(shard, global_index)

        global_index += len(shard)

        time2 = time.time()

        logger.info("TIMING: featurizing shard %d took %0.3f s" %

                    (shard_num, time2 - time1))

        yield X, y, w, ids

    return DiskDataset.create_dataset(shard_generator(), data_dir, self.tasks)

  def _get_shards(self, inputs, shard_size):

    """Break up input into shards.

    Parameters

    ----------

    inputs: list[object]

      Each entry in this list must be of the form `(featurization_input,

      label, weight, id)` or `(featurization_input, label, weight)` or

      `(featurization_input, label)` or `featurization_input` for one

      datapoint, where `featurization_input` is any input that is recognized

      by `self.featurizer`.

    shard_size: int

      The size of shard to generate.

    Returns

    -------

    Iterator which iterates over shards of data.

    """

    current_shard = []

    for i, datapoint in enumerate(inputs):

      if i != 0 and i % shard_size == 0:

        shard_data = current_shard

        current_shard = []

        yield shard_data

      current_shard.append(datapoint)

    yield current_shard

  def _featurize_shard(self, shard, global_index):

    """Featurizes a shard of an input data.

    Parameters

    ----------

    shard: list

      List each entry of which must be of the form `(featurization_input,

      label, weight, id)` or `(featurization_input, label, weight)` or

      `(featurization_input, label)` or `featurization_input` for one

      datapoint, where `featurization_input` is any input that is recognized

      by `self.featurizer`.

    global_index: int

      The starting index for this shard in the full set of provided inputs

    """

    features = []

    labels = []

    weights = []

    ids = []

    n_tasks = len(self.tasks)

    for i, entry in enumerate(shard):

      if not isinstance(entry, tuple):

        entry = (entry,)

      if len(entry) > 4:

        raise ValueError(

            "Entry is malformed and must be of length 1-4 containing featurization_input and optionally label, weight, and id."

        )

      if len(entry) == 4:

        featurization_input, label, weight, entry_id = entry

      elif len(entry) == 3:

        featurization_input, label, weight = entry

        entry_id = global_index + i

      elif len(entry) == 2:

        featurization_input, label = entry

        weight = np.ones((n_tasks), np.float32)

        entry_id = global_index + i

      elif len(entry) == 1:

        featurization_input = entry

        label = np.zeros((n_tasks), np.float32)

        weight = np.zeros((n_tasks), np.float32)

        entry_id = global_index + i

      feature = self.featurizer(featurization_input)

      features.append(feature)

      weights.append(weight)

      labels.append(label)

      ids.append(entry_id)

    X = np.concatenate(features, axis=0)

    y = np.array(labels)

    w = np.array(weights)

    ids = np.array(ids)

    return X, y, w, ids

        for i in order:

          yield (self._X[i], self._y[i], self._w[i], self._ids[i])

    class TorchDataset(torch.utils.data.IterableDataset):

    class TorchDataset(torch.utils.data.IterableDataset):  # type: ignore

      def __iter__(self):

        return iterate()

    Gets learning tasks associated with this dataset.

    """

    return self.tasks

    # if not len(self.metadata_df):

    #  raise ValueError("No data in dataset.")

    # return next(self.metadata_df.iterrows())[1]['task_names']

  def reshard(self, shard_size: int) -> None:

    """Reshards data to have specified shard size."""

    # Create temp directory to store resharded version

    reshard_dir = tempfile.mkdtemp()

    n_shards = self.get_number_shards()

    # Write data in new shards

    def generator():

      tasks = self.get_task_names()

      y_next = np.zeros((0,) + (len(tasks),))

      w_next = np.zeros((0,) + (len(tasks),))

      ids_next = np.zeros((0,), dtype=object)

      for (X, y, w, ids) in self.itershards():

      for shard_num, (X, y, w, ids) in enumerate(self.itershards()):

        logger.info("Resharding shard %d/%d" % (shard_num, n_shards))

        X_next = np.concatenate([X_next, X], axis=0)

        y_next = np.concatenate([y_next, y], axis=0)

        w_next = np.concatenate([w_next, w], axis=0)

      out_dir = tempfile.mkdtemp()

    tasks = self.get_task_names()

    n_shards = self.get_number_shards()

    def generator():

      for shard_num, row in self.metadata_df.iterrows():

        logger.info("Transforming shard %d/%d" % (shard_num, n_shards))

        X, y, w, ids = self.get_shard(shard_num)

        newx, newy, neww = fn(X, y, w)

        yield (newx, newy, neww, ids)

          for i in range(X.shape[0]):

            yield (X[i], y[i], w[i], ids[i])

    class TorchDataset(torch.utils.data.IterableDataset):

    class TorchDataset(torch.utils.data.IterableDataset):  # type: ignore

      def __iter__(self):

        return iterate()

    def generator():

      for ind, dataset in enumerate(datasets):

        logger.info("Merging in dataset %d/%d" % (ind, len(datasets)))

        X, y, w, ids = (dataset.X, dataset.y, dataset.w, dataset.ids)

        yield (X, y, w, ids)

    indices = np.array(sorted(indices)).astype(int)

    tasks = self.get_task_names()

    n_shards = self.get_number_shards()

    def generator():

      count, indices_count = 0, 0

      for shard_num, (X, y, w, ids) in enumerate(self.itershards()):

        logger.info("Selecting from shard %d/%d" % (shard_num, n_shards))

        shard_len = len(X)

        # Find indices which rest in this shard

        num_shard_elts = 0

    self._X_shape = self._find_array_shape(X)

    self._y_shape = self._find_array_shape(y)

    if w is None:

      if len(self._y_shape) == 1:

      if len(self._y_shape) == 0:

        # Case n_samples should be 1

        if n_samples != 1:

          raise ValueError("y can only be a scalar if n_samples == 1")

        w = np.ones_like(y)

      elif len(self._y_shape) == 1:

        w = np.ones(self._y_shape[0], np.float32)

      else:

        w = np.ones((self._y_shape[0], 1), np.float32)

          yield (get_image(self._X, i), get_image(self._y, i), self._w[i],

                 self._ids[i])

    class TorchDataset(torch.utils.data.IterableDataset):

    class TorchDataset(torch.utils.data.IterableDataset):  # type: ignore

      def __iter__(self):

        return iterate()

from scipy import misc

import deepchem as dc

import zipfile

import numpy as np