Merge pull request #1151 from miaecle/prog

from deepchem.models.tensorgraph.IRV import TensorflowMultiTaskIRVClassifier

from deepchem.models.tensorgraph.robust_multitask import RobustMultitaskClassifier

from deepchem.models.tensorgraph.robust_multitask import RobustMultitaskRegressor

from deepchem.models.tensorgraph.progressive_multitask import ProgressiveMultitaskRegressor

from deepchem.models.tensorgraph.progressive_multitask import ProgressiveMultitaskRegressor, ProgressiveMultitaskClassifier

from deepchem.models.tensorgraph.models.graph_models import WeaveTensorGraph, DTNNTensorGraph, DAGTensorGraph, GraphConvModel, MPNNTensorGraph

from deepchem.models.tensorgraph.models.symmetry_function_regression import BPSymmetryFunctionRegression, ANIRegression

from deepchem.utils.save import log

from deepchem.models.tensorgraph.tensor_graph import TensorGraph

from deepchem.models.tensorgraph.layers import Layer, SigmoidCrossEntropy, \

    Sigmoid, Feature, Label, Weights, Concat, WeightedError

    Sigmoid, Feature, Label, Weights, Concat, WeightedError, Stack

from deepchem.models.tensorgraph.layers import convert_to_layers

from deepchem.trans import undo_transforms

       https://www.ncbi.nlm.nih.gov/pmc/articles/PMC2750043/

    """

    self.mol_features = Feature(shape=(None, self.n_features))

    self._labels = Label(shape=(None, self.n_tasks))

    self._weights = Weights(shape=(None, self.n_tasks))

    predictions = IRVLayer(self.n_tasks, self.K, in_layers=[self.mol_features])

    costs = []

    self.labels_fd = []

    outputs = []

    for task in range(self.n_tasks):

      task_output = Slice(task, 1, in_layers=[predictions])

      sigmoid = Sigmoid(in_layers=[task_output])

      self.add_output(sigmoid)

      outputs.append(sigmoid)

      label = Label(shape=(None, 1))

      self.labels_fd.append(label)

      label = Slice(task, axis=1, in_layers=[self._labels])

      cost = SigmoidCrossEntropy(in_layers=[label, task_output])

      costs.append(cost)

    all_cost = Concat(in_layers=costs, axis=1)

    self.weights = Weights(shape=(None, self.n_tasks))

    loss = WeightedError(in_layers=[all_cost, self.weights]) + \

    loss = WeightedError(in_layers=[all_cost, self._weights]) + \

        IRVRegularize(predictions, self.penalty, in_layers=[predictions])

    self.set_loss(loss)

  def default_generator(self,

                        dataset,

                        epochs=1,

                        predict=False,

                        deterministic=True,

                        pad_batches=True):

    """TensorGraph style implementation """

    for epoch in range(epochs):

      if not predict:

        logger.info('Starting epoch %i' % epoch)

      for (X_b, y_b, w_b, ids_b) in dataset.iterbatches(

          batch_size=self.batch_size,

          deterministic=deterministic,

          pad_batches=pad_batches):

        feed_dict = dict()

        if y_b is not None:

          for index, label in enumerate(self.labels_fd):

            feed_dict[label] = y_b[:, index:index + 1]

        if w_b is not None:

          feed_dict[self.weights] = w_b

        feed_dict[self.mol_features] = X_b

        yield feed_dict

    outputs = Stack(axis=1, in_layers=outputs)

    self.add_output(outputs)

  def predict(self, dataset, transformers=[], outputs=None):

    out = super(TensorflowMultiTaskIRVClassifier, self).predict(

        dataset, transformers=transformers, outputs=outputs)

    out = np.concatenate(out, axis=1)

    out = np.round(out).astype(int)

    return out

  def predict_proba(self, dataset, transformers=[], outputs=None):

    out = super(TensorflowMultiTaskIRVClassifier, self).predict_proba(

        dataset, transformers=transformers, outputs=outputs)

    out = np.concatenate(out, axis=1)

    out = np.stack([1 - out, out], axis=2)

    return out

    return np.concatenate([1 - out, out], axis=2)

from deepchem.metrics import from_one_hot

from deepchem.models.tensorgraph.tensor_graph import TensorGraph, TFWrapper

from deepchem.models.tensorgraph.layers import Layer, Feature, Label, Weights, \

    WeightedError, Dense, Dropout, WeightDecay, Reshape, SoftMaxCrossEntropy, \

    L2Loss, ReduceSum, Concat, Stack, TensorWrapper, ReLU

    WeightedError, Dense, Dropout, WeightDecay, Reshape, SparseSoftMaxCrossEntropy, \

    L2Loss, ReduceSum, Concat, Stack, TensorWrapper, ReLU, Squeeze, SoftMax, Cast

from deepchem.models.tensorgraph.IRV import Slice

class ProgressiveMultitaskRegressor(TensorGraph):

  """Implements a progressive multitask neural network.

  """Implements a progressive multitask neural network for regression.

  Progressive Networks: https://arxiv.org/pdf/1606.04671v3.pdf

               weight_decay_penalty_type="l2",

               dropouts=0.5,

               activation_fns=tf.nn.relu,

               n_outputs=1,

               **kwargs):

    """Creates a progressive network.

    self.bias_init_consts = bias_init_consts

    self.dropouts = dropouts

    self.activation_fns = activation_fns

    self.n_outputs = n_outputs

    n_layers = len(layer_sizes)

    if not isinstance(weight_init_stddevs, collections.Sequence):

    # Add the input features.

    self.mol_features = Feature(shape=(None, n_features))

    self._task_labels = Label(shape=(None, n_tasks))

    self._task_weights = Weights(shape=(None, n_tasks))

    all_layers = {}

    outputs = []

      prev_layer = all_layers[(n_layers - 1, task)]

      layer = Dense(

          in_layers=[prev_layer],

          out_channels=1,

          out_channels=n_outputs,

          weights_initializer=TFWrapper(

              tf.truncated_normal_initializer,

              stddev=self.weight_init_stddevs[-1]),

                                                       n_layers)

        task_layers.extend(trainables)

        layer = layer + lateral_contrib

      outputs.append(layer)

      self.add_output(layer)

      task_label = Label(shape=(None, 1))

      task_weight = Weights(shape=(None, 1))

      weighted_loss = ReduceSum(

          L2Loss(in_layers=[task_label, layer, task_weight]))

      self.create_submodel(

          layers=task_layers, loss=weighted_loss, optimizer=None)

      output_layer = self.create_output(layer)

      outputs.append(output_layer)

      label = Slice(task, axis=1, in_layers=[self._task_labels])

      weight = Slice(task, axis=1, in_layers=[self._task_weights])

      task_loss = self.create_loss(layer, label, weight)

      self.create_submodel(layers=task_layers, loss=task_loss, optimizer=None)

    outputs = Stack(axis=1, in_layers=outputs)

    self.add_output(outputs)

    # Weight decay not activated

    """

    if weight_decay_penalty != 0.0:

          in_layers=[weighted_loss])

    """

  def create_loss(self, layer, label, weight):

    weighted_loss = ReduceSum(L2Loss(in_layers=[label, layer, weight]))

    return weighted_loss

  def create_output(self, layer):

    return layer

  def add_adapter(self, all_layers, task, layer_num):

    """Add an adapter connection for given task/layer combo"""

    i = layer_num

      weight_init_stddev = self.weight_init_stddevs[i]

      bias_init_const = self.bias_init_consts[i]

    elif i == len(self.layer_sizes):

      layer_sizes = self.layer_sizes + [1]

      layer_sizes = self.layer_sizes + [self.n_outputs]

      alpha_init_stddev = self.alpha_init_stddevs[-1]

      weight_init_stddev = self.weight_init_stddevs[-1]

      bias_init_const = self.bias_init_consts[-1]

    return dense2, trainable_layers

  def default_generator(self,

                        dataset,

                        epochs=1,

                        predict=False,

                        deterministic=True,

                        pad_batches=True):

    for epoch in range(epochs):

      for (X_b, y_b, w_b, ids_b) in dataset.iterbatches(

          batch_size=self.batch_size,

          deterministic=deterministic,

          pad_batches=pad_batches):

        feed_dict = dict()

        if X_b is not None:

          feed_dict[self.features[0]] = X_b

        if y_b is not None and not predict:

          for task in range(self.n_tasks):

            feed_dict[self.labels[task]] = y_b[:, task:task + 1]

        if w_b is not None and not predict:

          for task in range(self.n_tasks):

            feed_dict[self.task_weights[task]] = w_b[:, task:task + 1]

        yield feed_dict

  def fit(self,

          dataset,

          nb_epoch=10,

    self.fit_generator(generator, max_checkpoints_to_keep, checkpoint_interval,

                       restore, self.submodels[submodel])

  def predict_proba(self, dataset, transformers=[], outputs=None):

    return self.predict(dataset, transformers=transformers, outputs=outputs)

  def predict(self, dataset, transformers=[], outputs=None):

    retval = super(ProgressiveMultitaskRegressor, self).predict(

        dataset, transformers, outputs)

    # Results is of shape (n_samples, n_tasks, 1)

    out = np.stack(retval, axis=1)

    return out

class ProgressiveMultitaskClassifier(ProgressiveMultitaskRegressor):

  """Implements a progressive multitask neural network for classification.

  Progressive Networks: https://arxiv.org/pdf/1606.04671v3.pdf

  Progressive networks allow for multitask learning where each task

  gets a new column of weights. As a result, there is no exponential

  forgetting where previous tasks are ignored.

  """

  def __init__(self,

               n_tasks,

               n_features,

               alpha_init_stddevs=0.02,

               layer_sizes=[1000],

               weight_init_stddevs=0.02,

               bias_init_consts=1.0,

               weight_decay_penalty=0.0,

               weight_decay_penalty_type="l2",

               dropouts=0.5,

               activation_fns=tf.nn.relu,

               **kwargs):

    n_outputs = 2

    super(ProgressiveMultitaskClassifier, self).__init__(

        n_tasks,

        n_features,

        alpha_init_stddevs=alpha_init_stddevs,

        layer_sizes=layer_sizes,

        weight_init_stddevs=weight_init_stddevs,

        bias_init_consts=bias_init_consts,

        weight_decay_penalty=weight_decay_penalty,

        weight_decay_penalty_type=weight_decay_penalty_type,

        dropouts=dropouts,

        activation_fns=activation_fns,

        n_outputs=n_outputs,

        **kwargs)

  def create_loss(self, layer, label, weight):

    task_label = Squeeze(squeeze_dims=1, in_layers=[label])

    task_label = Cast(dtype=tf.int32, in_layers=[task_label])

    task_weight = Squeeze(squeeze_dims=1, in_layers=[weight])

    loss = SparseSoftMaxCrossEntropy(in_layers=[task_label, layer])

    weighted_loss = WeightedError(in_layers=[loss, task_weight])

    return weighted_loss

  def create_output(self, layer):

    output = SoftMax(in_layers=[layer])

    return output

    results = estimator.evaluate(input_fn=lambda: input_fn(1))

    assert results['loss'] < 1e-2

    assert results['error'] < 0.1

  def test_irv(self):

    """Test creating an Estimator from a IRVClassifier."""

    n_samples = 50

    n_features = 3

    n_tasks = 2

    # Create a dataset and an input function for processing it.

    np.random.seed(123)

    X = np.random.rand(n_samples, n_features)

    y = np.zeros((n_samples, n_tasks))

    dataset = dc.data.NumpyDataset(X, y)

    transformers = [dc.trans.IRVTransformer(10, n_tasks, dataset)]

    for transformer in transformers:

      dataset = transformer.transform(dataset)

    def input_fn(epochs):

      x, y, weights = dataset.make_iterator(

          batch_size=n_samples, epochs=epochs).get_next()

      return {'x': x, 'weights': weights}, y

    # Create a TensorGraph model.

    model = dc.models.TensorflowMultiTaskIRVClassifier(

        n_tasks, K=10, learning_rate=0.001, penalty=0.05, batch_size=50)

    model.build()

    # Create an estimator from it.

    x_col = tf.feature_column.numeric_column('x', shape=(2 * 10 * n_tasks,))

    weight_col = tf.feature_column.numeric_column('weights', shape=(n_tasks,))

    def accuracy(labels, predictions, weights):

      return tf.metrics.accuracy(labels, tf.round(predictions), weights)

    metrics = {'accuracy': accuracy}

    estimator = model.make_estimator(

        feature_columns=[x_col], weight_column=weight_col, metrics=metrics)

    # Train the model.

    estimator.train(input_fn=lambda: input_fn(100))

    # Evaluate the model.

    results = estimator.evaluate(input_fn=lambda: input_fn(1))

    assert results['accuracy'] > 0.9

    assert scores[regression_metric.name] > .9

  def test_tf_progressive_classification_overfit(self):

    """Test tf progressive multitask overfits tiny data."""

    np.random.seed(123)

    n_tasks = 5

    n_samples = 10

    n_features = 3

    # Generate dummy dataset

    np.random.seed(123)

    ids = np.arange(n_samples)

    X = np.random.rand(n_samples, n_features)

    y = np.ones((n_samples, n_tasks))

    w = np.ones((n_samples, n_tasks))

    dataset = dc.data.NumpyDataset(X, y, w, ids)

    metric = dc.metrics.Metric(dc.metrics.accuracy_score, task_averager=np.mean)

    model = dc.models.ProgressiveMultitaskClassifier(

        n_tasks,

        n_features,

        layer_sizes=[50],

        bypass_layer_sizes=[10],

        dropouts=[0.],

        learning_rate=0.003,

        weight_init_stddevs=[.1],

        alpha_init_stddevs=[.02],

        batch_size=n_samples,

        use_queue=False)

    # Fit trained model

    model.fit(dataset, nb_epoch=20)

    # Eval model on train

    scores = model.evaluate(dataset, [metric])

    assert scores[metric.name] > .9

  def test_tf_progressive_regression_overfit(self):

    """Test tf progressive multitask overfits tiny data."""

    np.random.seed(123)