Commit a1a2b847 authored by Bharath Ramsundar's avatar Bharath Ramsundar
Browse files

Fixes for metric issues

parent 84ce3736

deepchem/data/__init__.py

+1 −0
Original line number Diff line number Diff line
@@ -8,3 +8,4 @@ from __future__ import unicode_literals 
# TODO(rbharath): Get rid of * import

from deepchem.data.datasets import *

from deepchem.data.supports import *

import deepchem.data.tests

deepchem/data/datasets.py

+24 −0
Original line number Diff line number Diff line
@@ -567,6 +567,12 @@ class DiskDataset(Dataset): 
      for i in range(num_shards):

        X, y, w, ids = dataset.get_shard(shard_perm[i])

        n_samples = X.shape[0]

        #################################################################### DEBUG

        # TODO(rbharath): This happens in tests sometimes, but don't understand why?

        # Handle edge case.

        if n_samples == 0:

          continue

        #################################################################### DEBUG

        if not deterministic:

          sample_perm = np.random.permutation(n_samples)

        else:
@@ -575,6 +581,11 @@ class DiskDataset(Dataset): 
          shard_batch_size = n_samples

        else:

          shard_batch_size = batch_size 

        #################################################################### DEBUG

        #print("iterbatch()")

        #print("n_samples, shard_batch_size")

        #print(n_samples, shard_batch_size)

        #################################################################### DEBUG

        interval_points = np.linspace(

            0, n_samples, np.ceil(float(n_samples)/shard_batch_size)+1, dtype=int)

        for j in range(len(interval_points)-1):
@@ -935,10 +946,20 @@ class DiskDataset(Dataset): 
      shard_len = len(X)

      # Find indices which rest in this shard

      num_shard_elts = 0

      ############################################################### DEBUG

      #print("Dataset.select()")

      #print("shard_num, shard_len, indices_count, indices_count+num_shard_elts, count, count+shard_len")

      #print(shard_num, shard_len, indices_count, indices_count+num_shard_elts, count, count+shard_len)

      ############################################################### DEBUG

      while indices[indices_count+num_shard_elts] < count + shard_len:

        num_shard_elts += 1

        if indices_count + num_shard_elts >= len(indices):

          break

      ############################################################### DEBUG

      #print("after while loop")

      #print("num_shard_elts")

      #print(num_shard_elts)

      ############################################################### DEBUG

      # Need to offset indices to fit within shard_size

      shard_inds =  indices[indices_count:indices_count+num_shard_elts] - count

      X_sel = X[shard_inds]
@@ -954,6 +975,9 @@ class DiskDataset(Dataset): 
      # Updating counts

      indices_count += num_shard_elts

      count += shard_len

      # Break when all indices have been used up already

      if indices_count >= len(indices):

        break

    return DiskDataset(data_dir=select_dir,

                   metadata_rows=metadata_rows,

                   verbosity=self.verbosity)

deepchem/data/supports.py

+4 −32
Original line number Diff line number Diff line
@@ -138,31 +138,6 @@ def get_single_task_support(dataset, n_pos, n_neg, task, replace=True): 
    List of NumpyDatasets, each of which is a support set.

  """

  return get_task_support(dataset, 1, n_pos, n_neg, task)[0]

  #y_task = dataset.y[:, task]



  ## Split data into pos and neg lists.

  #pos_mols = np.where(y_task == 1)[0]

  #neg_mols = np.where(y_task == 0)[0]



  ## Get randomly sampled pos/neg indices (with replacement)

  #pos_inds = pos_mols[np.random.choice(len(pos_mols), (n_pos), replace=replace)]

  #neg_inds = neg_mols[np.random.choice(len(neg_mols), (n_neg), replace=replace)]



  ## Handle one-d vs. non one-d feature matrices

  #one_dimensional_features = (len(dataset.X.shape) == 1)

  #if not one_dimensional_features:

  #  X_trial = np.vstack(

  #      [dataset.X[pos_inds], dataset.X[neg_inds]])

  #else:

  #  X_trial = np.concatenate(

  #      [dataset.X[pos_inds], dataset.X[neg_inds]])

  #y_trial = np.concatenate(

  #    [dataset.y[pos_inds, task], dataset.y[neg_inds, task]])

  #w_trial = np.concatenate(

  #    [dataset.w[pos_inds, task], dataset.w[neg_inds, task]])

  #ids_trial = np.concatenate(

  #    [dataset.ids[pos_inds], dataset.ids[neg_inds]])

  #return NumpyDataset(X_trial, y_trial, w_trial, ids_trial)



def get_task_support(dataset, n_episodes, n_pos, n_neg, task, log_every_n=50):

  """Generates one support set purely for specified task.
@@ -299,28 +274,25 @@ class SupportGenerator(object): 
  Iterates over tasks and trials. For each trial, picks one support from

  each task, and returns in a randomized order

  """

  def __init__(self, dataset, tasks, n_pos, n_neg, n_trials, replace):

  def __init__(self, dataset, n_pos, n_neg, n_trials, replace):

    """

    Parameters

    ----------

    dataset: dc.data.Dataset

      Holds dataset from which support sets will be sampled.

    tasks: list

      Indices of tasks from which supports are sampled.

      TODO(rbharath): Can this be removed.

    n_pos: int

      Number of positive samples

    n_neg: int

      Number of negative samples.

    n_trials: int

      Number of passes over tasks to make. In total, n_tasks*n_trials

      Number of passes over dataset to make. In total, n_tasks*n_trials

      support sets will be sampled by algorithm.

    replace: bool

      Whether to use sampling with or without replacement.

    """

      

    self.tasks = tasks

    self.n_tasks = len(tasks)

    self.tasks = range(len(dataset.get_task_names()) )

    self.n_tasks = len(self.tasks)

    self.n_trials = n_trials

    self.dataset = dataset

    self.n_pos = n_pos

deepchem/data/tests/test_support_generator.py

+38 −2
Original line number Diff line number Diff line
@@ -119,7 +119,7 @@ class TestSupports(unittest.TestCase): 


    # Create support generator

    supp_gen = dc.data.SupportGenerator(

        dataset, np.arange(n_tasks), n_pos, n_neg, n_trials, replace=True)

        dataset, n_pos, n_neg, n_trials, replace=True)



  def test_simple_episode_generator(self):

    """Conducts simple test that episode generator runs."""
@@ -262,7 +262,7 @@ class TestSupports(unittest.TestCase): 


    # Create support generator

    supp_gen = dc.data.SupportGenerator(

        dataset, np.arange(n_tasks), n_pos, n_neg, n_trials, replace=True)

        dataset, n_pos, n_neg, n_trials, replace=True)

    num_supports = 0

    

    for (task, support) in supp_gen:
@@ -272,3 +272,39 @@ class TestSupports(unittest.TestCase): 
      n_supp_pos = np.count_nonzero(support.y)

      assert n_supp_pos == n_pos

    assert num_supports == n_trials



  def test_evaluation_strategy(self):

    """Tests that sampling supports for eval works properly."""

    n_samples = 2000

    n_features = 3

    n_tasks = 5

    n_pos = 1

    n_neg = 5 

    n_trials = 10

    

    # Generate dummy dataset

    np.random.seed(123)

    ids = np.arange(n_samples)

    X = np.random.rand(n_samples, n_features)

    y = np.random.randint(2, size=(n_samples, n_tasks))

    w = np.random.randint(2, size=(n_samples, n_tasks))

    dataset = dc.data.NumpyDataset(X, y, w, ids)



    support_generator = dc.data.SupportGenerator(dataset, 

        n_pos, n_neg, n_trials, replace=False)



    for ind, (task, support) in enumerate(support_generator):

      task_dataset = dc.data.get_task_dataset_minus_support(

          dataset, support, task)



      task_y = dataset.y[:, task]

      task_w = dataset.w[:, task]

      print("Number of y elements (including missing data)")

      print(len(task_y))

      task_y = task_y[task_w != 0]

      print("len(task_y), len(support), len(task_dataset)")

      print(len(task_y), len(support), len(task_dataset))

      assert len(task_y) == len(support) + len(task_dataset)

      print("Verifying that task_dataset doesn't overlap with support.")

      for task_id in task_dataset.ids:

        assert task_id not in set(support.ids)

deepchem/models/tf_keras_models/support_classifier.py

+26 −25
Original line number Diff line number Diff line
@@ -70,30 +70,19 @@ class SupportGraphClassifier(Model): 


  def get_training_op(self, loss):

    """Attaches an optimizer to the graph."""

    ################################################################# DEBUG

    #global_step = tf.Variable(0, trainable=False)

    #learning_rate = tf.train.exponential_decay(

    #    self.learning_rate, global_step,

    #    self.decay_steps, self.decay_rate, staircase=True)

    #opt = tf.train.AdamOptimizer(learning_rate)

    ## Get train function

    #return opt.minimize(self.loss_op, name="train", global_step=global_step)

    opt = tf.train.AdamOptimizer(self.learning_rate)

    return opt.minimize(self.loss_op, name="train")

    ################################################################# DEBUG



  def add_placeholders(self):

    """Adds placeholders to graph."""

    self.test_label_placeholder = Input(

        #tensor=K.placeholder(shape=(self.test_batch_size), dtype='float32',

        tensor=K.placeholder(shape=(self.test_batch_size), dtype='float32',

        name="label_placeholder"))

    self.test_weight_placeholder = Input(

        #tensor=K.placeholder(shape=(self.test_batch_size), dtype='float32',

        tensor=K.placeholder(shape=(self.test_batch_size), dtype='float32',

        name="weight_placeholder"))



    # TODO(rbharath): There should be weights for the support being used! 

    # TODO(rbharath): Should weights for the support be used?

    # Support labels

    self.support_label_placeholder = Input(

        tensor=K.placeholder(shape=[self.support_batch_size], dtype='float32',
@@ -222,6 +211,10 @@ class SupportGraphClassifier(Model): 
      for ind, (task, support, test) in enumerate(episode_generator):

        if ind % log_every_n_samples == 0:

          print("Epoch %d, Sample %d from task %s" % (epoch, ind, str(task)))

          ############################################################### DEBUG

          print("task, len(support), len(test)")

          print(task, len(support), len(test))

          ############################################################### DEBUG

        # Get batch to try it out on

        feed_start = time.time()

        feed_dict = self.construct_feed_dict(test, support)
@@ -377,25 +370,15 @@ class SupportGraphClassifier(Model): 
    ########################################################### DEBUG

    # pred corresponds to prob(example == 1) 

    y_pred_batch = np.zeros((n_samples, 2))

    # Remove padded elements

    pred = pred[:n_samples]

    y_pred_batch[:, 1] = pred

    y_pred_batch[:, 0] = 1-pred

    print("scores[:5]")

    print(scores[:5])

    print("pred[:5]")

    print(pred[:5])

    print("y_pred_batch[:5]")

    print(y_pred_batch[:5])

    ########################################################### DEBUG

    #y_pred_batch = to_one_hot(np.round(pred))

    ########################################################### DEBUG

    # Remove padded elements

    #y_pred_batch = y_pred_batch[:n_samples]

    ########################################################### DEBUG

    return y_pred_batch

    

  def evaluate(self, dataset, metric, n_pos=1,

               n_neg=9, n_trials=1000, exclude_support=True, replace=True):

               n_neg=9, n_trials=1000, exclude_support=True):

    """Evaluate performance on dataset according to metrics
@@ -433,9 +416,14 @@ class SupportGraphClassifier(Model): 
    test_tasks = range(len(dataset.get_task_names()))

    task_scores = {task: [] for task in test_tasks}

    support_generator = SupportGenerator(dataset, test_tasks,

        n_pos, n_neg, n_trials, replace)

        n_pos, n_neg, n_trials)

    for ind, (task, support) in enumerate(support_generator):

      print("Eval sample %d from task %s" % (ind, str(task)))

      ################################################################ DEBUG

      print("task, len(support)")

      print(task, len(support))

      ################################################################ DEBUG

      

      # TODO(rbharath): Add test for get_task_dataset_minus_support for

      # multitask case with missing data...

      if exclude_support:
@@ -446,6 +434,19 @@ class SupportGraphClassifier(Model): 
        task_dataset = get_task_dataset(dataset, task)

      y_pred = self.predict_proba(support, task_dataset)

      ################################################################ DEBUG

      task_y = dataset.y[:, task]

      task_w = dataset.w[:, task]

      print("Number of y elements (including missing data)")

      print(len(task_y))

      task_y = task_y[task_w != 0]

      print("Number of y elements (excluding missing data)")

      print(len(task_y))

      print("len(task_y), len(support), len(task_dataset)")

      print(len(task_y), len(support), len(task_dataset))

      assert len(task_y) == len(support) + len(task_dataset)

      print("Verifying that task_dataset doesn't overlap with support.")

      for task_id in task_dataset.ids:

        assert task_id not in set(support.ids)

      print("evaluate()")

      y_pred_hot = from_one_hot(y_pred)

      print("y_pred")

CRA Git | Maintained and supported by SUSTech CRA and CCSE