Loading deepchem/models/tf_new_models/support_classifier.py +65 −65 Original line number Diff line number Diff line Loading @@ -100,71 +100,71 @@ class SupportGraphClassifier(Model): feed_dict[self.phase] = training return feed_dict def old_fit(self, dataset, n_trials=1000, n_steps_per_trial=1, n_pos=1, n_neg=9, log_every_n_samples=10, replace=True, **kwargs): """Fits model on dataset. Note that fitting for support models is quite different from fitting for other deep models. Fitting is a two-level process. We perform n_trials, where for each trial, we randomply sample a support set for each given task, and independently a test set from that same task. The SupportGenerator class iterates over the tasks in random order. Parameters ---------- dataset: dc.data.Dataset Dataset to fit model on. n_trials: int, optional Number of (support, test) pairs to sample and train on. n_steps_per_trial: int, optional Number of gradient descent steps to take per support. n_pos: int, optional Number of positive examples per support. n_neg: int, optional Number of negative examples per support. log_every_n_samples: int, optional Displays info every this number of samples replace: bool, optional Whether or not to use replacement when sampling supports/tests. """ time_start = time.time() # Perform the optimization n_tasks = len(dataset.get_task_names()) feed_total, run_total, test_total = 0, 0, 0 # Create different support sets support_generator = SupportGenerator(dataset, range(n_tasks), n_pos, n_neg, n_trials) recent_losses = [] for ind, (task, support) in enumerate(support_generator): if ind % log_every_n_samples == 0: print("Sample %d from task %s" % (ind, str(task))) # Get batch to try it out on test_start = time.time() test = get_single_task_test(dataset, self.test_batch_size, task, replace) test_end = time.time() test_total += (test_end - test_start) feed_start = time.time() feed_dict = self.construct_feed_dict(test, support) feed_end = time.time() feed_total += (feed_end - feed_start) for step in range(n_steps_per_trial): # Train on support set, batch pair run_start = time.time() _, loss = self.sess.run([self.train_op, self.loss_op], feed_dict=feed_dict) run_end = time.time() run_total += (run_end - run_start) if ind % log_every_n_samples == 0: mean_loss = np.mean(np.array(recent_losses)) print("\tmean loss is %s" % str(mean_loss)) recent_losses = [] else: recent_losses.append(loss) time_end = time.time() print("old_fit took %s seconds" % str(time_end-time_start)) print("test_total: %s" % str(test_total)) print("feed_total: %s" % str(feed_total)) print("run_total: %s" % str(run_total)) #def old_fit(self, dataset, n_trials=1000, n_steps_per_trial=1, n_pos=1, # n_neg=9, log_every_n_samples=10, replace=True, **kwargs): # """Fits model on dataset. # Note that fitting for support models is quite different from fitting for # other deep models. Fitting is a two-level process. We perform n_trials, # where for each trial, we randomply sample a support set for each given # task, and independently a test set from that same task. The # SupportGenerator class iterates over the tasks in random order. # Parameters # ---------- # dataset: dc.data.Dataset # Dataset to fit model on. # n_trials: int, optional # Number of (support, test) pairs to sample and train on. # n_steps_per_trial: int, optional # Number of gradient descent steps to take per support. # n_pos: int, optional # Number of positive examples per support. # n_neg: int, optional # Number of negative examples per support. # log_every_n_samples: int, optional # Displays info every this number of samples # replace: bool, optional # Whether or not to use replacement when sampling supports/tests. # """ # time_start = time.time() # # Perform the optimization # n_tasks = len(dataset.get_task_names()) # feed_total, run_total, test_total = 0, 0, 0 # # Create different support sets # support_generator = SupportGenerator(dataset, range(n_tasks), # n_pos, n_neg, n_trials) # recent_losses = [] # for ind, (task, support) in enumerate(support_generator): # if ind % log_every_n_samples == 0: # print("Sample %d from task %s" % (ind, str(task))) # # Get batch to try it out on # test_start = time.time() # test = get_single_task_test(dataset, self.test_batch_size, task, replace) # test_end = time.time() # test_total += (test_end - test_start) # feed_start = time.time() # feed_dict = self.construct_feed_dict(test, support) # feed_end = time.time() # feed_total += (feed_end - feed_start) # for step in range(n_steps_per_trial): # # Train on support set, batch pair # run_start = time.time() # _, loss = self.sess.run([self.train_op, self.loss_op], feed_dict=feed_dict) # run_end = time.time() # run_total += (run_end - run_start) # if ind % log_every_n_samples == 0: # mean_loss = np.mean(np.array(recent_losses)) # print("\tmean loss is %s" % str(mean_loss)) # recent_losses = [] # else: # recent_losses.append(loss) # time_end = time.time() # print("old_fit took %s seconds" % str(time_end-time_start)) # print("test_total: %s" % str(test_total)) # print("feed_total: %s" % str(feed_total)) # print("run_total: %s" % str(run_total)) def fit(self, dataset, n_episodes_per_epoch=1000, nb_epochs=1, n_pos=1, n_neg=9, log_every_n_samples=10, **kwargs): Loading deepchem/nn/layers.py +3 −4 Original line number Diff line number Diff line Loading @@ -702,9 +702,6 @@ def cos(x, y): class LSTMStep(Layer): """ LSTM whose call is a single step in the LSTM. TODO(rbharath): forget_bias_init uses get_value(), which evaluates provided tensors in session. Seems quite unnecessary... This layer exists because the Keras LSTM layer is intrinsically linked to an RNN with sequence inputs, and here, we will not be using sequence inputs, but rather we generate a sequence of inputs using the intermediate outputs of the Loading @@ -721,6 +718,8 @@ class LSTMStep(Layer): self.init = initializations.get(init) self.inner_init = initializations.get(inner_init) # No other forget biases supported right now. assert forget_bias_init == "one" self.forget_bias_init = initializations.get(forget_bias_init) self.activation = activations.get(activation) self.inner_activation = activations.get(inner_activation) Loading @@ -737,7 +736,7 @@ class LSTMStep(Layer): self.b = model_ops.variable(np.hstack( (np.zeros(self.output_dim), model_ops.get_value(self.forget_bias_init((self.output_dim,))), np.ones(self.output_dim), np.zeros(self.output_dim), np.zeros(self.output_dim)))) self.trainable_weights = [self.W, self.U, self.b] Loading deepchem/nn/model_ops.py +8 −101 Original line number Diff line number Diff line Loading @@ -17,13 +17,6 @@ from collections import defaultdict py_all = all # TODO(rbharath): REMOVE GLOBAL VARS! BREAKS DEEPCHEM STYLE! # This is the default internal TF session used by Keras. # It can be set manually via `set_session(sess)`. _SESSION = None # This boolean flag can be set to True to leave variable initialization # up to the user. # Change its value via `manual_variable_initialization(value)`. _MANUAL_VAR_INIT = True _UID_PREFIXES = defaultdict(int) # This dictionary holds a mapping {graph: learning_phase}. # A learning phase is a bool tensor used to run Keras models in Loading Loading @@ -151,40 +144,18 @@ def normalize_batch_in_training(x, gamma, beta, epsilon) return normed, mean, var def eval(x): """Evaluates the value of a variable. Returns a Numpy array. Parameters ---------- x: A variable. Returns ------- A Numpy array. """ return x.eval(session=get_session()) def ones(shape, dtype=None, name=None): """Instantiates an all-ones tensor variable and returns it. # Arguments Parameters ---------- shape: Tuple of integers, shape of returned Keras variable. dtype: String, data type of returned Keras variable. name: String, name of returned Keras variable. # Returns Returns ------- A Keras variable, filled with `1.0`. # Example ```python >>> from keras import backend as K >>> kvar = K.ones((3,4)) >>> K.eval(kvar) array([[ 1., 1., 1., 1.], [ 1., 1., 1., 1.], [ 1., 1., 1., 1.]], dtype=float32) ``` """ if dtype is None: dtype = tf.float32 Loading @@ -196,10 +167,12 @@ def ones(shape, dtype=None, name=None): def cast_to_floatx(x): """Cast a Numpy array to the default Keras float type. # Arguments Parameters ---------- x: Numpy array. # Returns Returns ------- The same Numpy array, cast to its new type. """ return np.asarray(x, dtype=tf.float32) Loading Loading @@ -227,19 +200,6 @@ def int_shape(x): shape = x.get_shape() return tuple([i.__int__() for i in shape]) def get_value(x): """Returns the value of a variable. Parameters ---------- x: input variable. Returns ------- A Numpy array. """ return x.eval(session=get_session()) def get_uid(prefix=''): """Provides a unique UID given a string prefix. Loading @@ -254,59 +214,6 @@ def get_uid(prefix=''): _UID_PREFIXES[prefix] += 1 return _UID_PREFIXES[prefix] def _initialize_variables(): if hasattr(tf, 'global_variables'): variables = tf.global_variables() else: variables = tf.all_variables() uninitialized_variables = [] for v in variables: if not hasattr(v, '_keras_initialized') or not v._keras_initialized: uninitialized_variables.append(v) v._keras_initialized = True if uninitialized_variables: sess = get_session() if hasattr(tf, 'variables_initializer'): sess.run(tf.variables_initializer(uninitialized_variables)) else: sess.run(tf.initialize_variables(uninitialized_variables)) # TODO(rbharath): DANGEROUS! THIS IS LEAKY AND BREAKS DEEPCHEM STYLE! def get_session(): """Returns the TF session to be used by the backend. If a default TensorFlow session is available, we will return it. Else, we will return the global Keras session. If no global Keras session exists at this point: we will create a new global session. Note that you can manually set the global session via `K.set_session(sess)`. Returns ------- A TensorFlow session. """ global _SESSION if tf.get_default_session() is not None: session = tf.get_default_session() else: if _SESSION is None: if not os.environ.get('OMP_NUM_THREADS'): config = tf.ConfigProto(allow_soft_placement=True) else: nb_thread = int(os.environ.get('OMP_NUM_THREADS')) config = tf.ConfigProto(intra_op_parallelism_threads=nb_thread, allow_soft_placement=True) _SESSION = tf.Session(config=config) session = _SESSION if not _MANUAL_VAR_INIT: _initialize_variables() return session def concatenate(tensors, axis=-1): """Concatenates a list of tensors alongside the specified axis. Loading deepchem/nn/regularizers.py +0 −35 Original line number Diff line number Diff line Loading @@ -17,41 +17,6 @@ class Regularizer(object): def __call__(self, x): return 0 class EigenvalueRegularizer(Regularizer): """Regularizer based on the eignvalues of a weight matrix. Only available for tensors of rank 2. # Arguments k: Float; modulates the amount of regularization to apply. """ def __init__(self, k): self.k = k def __call__(self, x): if model_ops.get_ndim(x) != 2: raise ValueError('EigenvalueRegularizer ' 'is only available for tensors of rank 2.') covariance = model_ops.dot(tf.transpose(x), x) dim1, dim2 = model_ops.eval(tf.shape(covariance)) # Power method for approximating the dominant eigenvector: power = 9 # Number of iterations of the power method. o = model_ops.ones([dim1, 1]) # Initial values for the dominant eigenvector. main_eigenvect = model_ops.dot(covariance, o) for n in range(power - 1): main_eigenvect = model_ops.dot(covariance, main_eigenvect) covariance_d = model_ops.dot(covariance, main_eigenvect) # The corresponding dominant eigenvalue: main_eigenval = (model_ops.dot(tf.transpose(covariance_d), main_eigenvect) / model_ops.dot(tf.transpose(main_eigenvect), main_eigenvect)) # Multiply by the given regularization gain. regularization = (main_eigenval ** 0.5) * self.k return model_ops.sum(regularization) class L1L2Regularizer(Regularizer): """Regularizer for L1 and L2 regularization. Loading Loading
deepchem/models/tf_new_models/support_classifier.py +65 −65 Original line number Diff line number Diff line Loading @@ -100,71 +100,71 @@ class SupportGraphClassifier(Model): feed_dict[self.phase] = training return feed_dict def old_fit(self, dataset, n_trials=1000, n_steps_per_trial=1, n_pos=1, n_neg=9, log_every_n_samples=10, replace=True, **kwargs): """Fits model on dataset. Note that fitting for support models is quite different from fitting for other deep models. Fitting is a two-level process. We perform n_trials, where for each trial, we randomply sample a support set for each given task, and independently a test set from that same task. The SupportGenerator class iterates over the tasks in random order. Parameters ---------- dataset: dc.data.Dataset Dataset to fit model on. n_trials: int, optional Number of (support, test) pairs to sample and train on. n_steps_per_trial: int, optional Number of gradient descent steps to take per support. n_pos: int, optional Number of positive examples per support. n_neg: int, optional Number of negative examples per support. log_every_n_samples: int, optional Displays info every this number of samples replace: bool, optional Whether or not to use replacement when sampling supports/tests. """ time_start = time.time() # Perform the optimization n_tasks = len(dataset.get_task_names()) feed_total, run_total, test_total = 0, 0, 0 # Create different support sets support_generator = SupportGenerator(dataset, range(n_tasks), n_pos, n_neg, n_trials) recent_losses = [] for ind, (task, support) in enumerate(support_generator): if ind % log_every_n_samples == 0: print("Sample %d from task %s" % (ind, str(task))) # Get batch to try it out on test_start = time.time() test = get_single_task_test(dataset, self.test_batch_size, task, replace) test_end = time.time() test_total += (test_end - test_start) feed_start = time.time() feed_dict = self.construct_feed_dict(test, support) feed_end = time.time() feed_total += (feed_end - feed_start) for step in range(n_steps_per_trial): # Train on support set, batch pair run_start = time.time() _, loss = self.sess.run([self.train_op, self.loss_op], feed_dict=feed_dict) run_end = time.time() run_total += (run_end - run_start) if ind % log_every_n_samples == 0: mean_loss = np.mean(np.array(recent_losses)) print("\tmean loss is %s" % str(mean_loss)) recent_losses = [] else: recent_losses.append(loss) time_end = time.time() print("old_fit took %s seconds" % str(time_end-time_start)) print("test_total: %s" % str(test_total)) print("feed_total: %s" % str(feed_total)) print("run_total: %s" % str(run_total)) #def old_fit(self, dataset, n_trials=1000, n_steps_per_trial=1, n_pos=1, # n_neg=9, log_every_n_samples=10, replace=True, **kwargs): # """Fits model on dataset. # Note that fitting for support models is quite different from fitting for # other deep models. Fitting is a two-level process. We perform n_trials, # where for each trial, we randomply sample a support set for each given # task, and independently a test set from that same task. The # SupportGenerator class iterates over the tasks in random order. # Parameters # ---------- # dataset: dc.data.Dataset # Dataset to fit model on. # n_trials: int, optional # Number of (support, test) pairs to sample and train on. # n_steps_per_trial: int, optional # Number of gradient descent steps to take per support. # n_pos: int, optional # Number of positive examples per support. # n_neg: int, optional # Number of negative examples per support. # log_every_n_samples: int, optional # Displays info every this number of samples # replace: bool, optional # Whether or not to use replacement when sampling supports/tests. # """ # time_start = time.time() # # Perform the optimization # n_tasks = len(dataset.get_task_names()) # feed_total, run_total, test_total = 0, 0, 0 # # Create different support sets # support_generator = SupportGenerator(dataset, range(n_tasks), # n_pos, n_neg, n_trials) # recent_losses = [] # for ind, (task, support) in enumerate(support_generator): # if ind % log_every_n_samples == 0: # print("Sample %d from task %s" % (ind, str(task))) # # Get batch to try it out on # test_start = time.time() # test = get_single_task_test(dataset, self.test_batch_size, task, replace) # test_end = time.time() # test_total += (test_end - test_start) # feed_start = time.time() # feed_dict = self.construct_feed_dict(test, support) # feed_end = time.time() # feed_total += (feed_end - feed_start) # for step in range(n_steps_per_trial): # # Train on support set, batch pair # run_start = time.time() # _, loss = self.sess.run([self.train_op, self.loss_op], feed_dict=feed_dict) # run_end = time.time() # run_total += (run_end - run_start) # if ind % log_every_n_samples == 0: # mean_loss = np.mean(np.array(recent_losses)) # print("\tmean loss is %s" % str(mean_loss)) # recent_losses = [] # else: # recent_losses.append(loss) # time_end = time.time() # print("old_fit took %s seconds" % str(time_end-time_start)) # print("test_total: %s" % str(test_total)) # print("feed_total: %s" % str(feed_total)) # print("run_total: %s" % str(run_total)) def fit(self, dataset, n_episodes_per_epoch=1000, nb_epochs=1, n_pos=1, n_neg=9, log_every_n_samples=10, **kwargs): Loading
deepchem/nn/layers.py +3 −4 Original line number Diff line number Diff line Loading @@ -702,9 +702,6 @@ def cos(x, y): class LSTMStep(Layer): """ LSTM whose call is a single step in the LSTM. TODO(rbharath): forget_bias_init uses get_value(), which evaluates provided tensors in session. Seems quite unnecessary... This layer exists because the Keras LSTM layer is intrinsically linked to an RNN with sequence inputs, and here, we will not be using sequence inputs, but rather we generate a sequence of inputs using the intermediate outputs of the Loading @@ -721,6 +718,8 @@ class LSTMStep(Layer): self.init = initializations.get(init) self.inner_init = initializations.get(inner_init) # No other forget biases supported right now. assert forget_bias_init == "one" self.forget_bias_init = initializations.get(forget_bias_init) self.activation = activations.get(activation) self.inner_activation = activations.get(inner_activation) Loading @@ -737,7 +736,7 @@ class LSTMStep(Layer): self.b = model_ops.variable(np.hstack( (np.zeros(self.output_dim), model_ops.get_value(self.forget_bias_init((self.output_dim,))), np.ones(self.output_dim), np.zeros(self.output_dim), np.zeros(self.output_dim)))) self.trainable_weights = [self.W, self.U, self.b] Loading
deepchem/nn/model_ops.py +8 −101 Original line number Diff line number Diff line Loading @@ -17,13 +17,6 @@ from collections import defaultdict py_all = all # TODO(rbharath): REMOVE GLOBAL VARS! BREAKS DEEPCHEM STYLE! # This is the default internal TF session used by Keras. # It can be set manually via `set_session(sess)`. _SESSION = None # This boolean flag can be set to True to leave variable initialization # up to the user. # Change its value via `manual_variable_initialization(value)`. _MANUAL_VAR_INIT = True _UID_PREFIXES = defaultdict(int) # This dictionary holds a mapping {graph: learning_phase}. # A learning phase is a bool tensor used to run Keras models in Loading Loading @@ -151,40 +144,18 @@ def normalize_batch_in_training(x, gamma, beta, epsilon) return normed, mean, var def eval(x): """Evaluates the value of a variable. Returns a Numpy array. Parameters ---------- x: A variable. Returns ------- A Numpy array. """ return x.eval(session=get_session()) def ones(shape, dtype=None, name=None): """Instantiates an all-ones tensor variable and returns it. # Arguments Parameters ---------- shape: Tuple of integers, shape of returned Keras variable. dtype: String, data type of returned Keras variable. name: String, name of returned Keras variable. # Returns Returns ------- A Keras variable, filled with `1.0`. # Example ```python >>> from keras import backend as K >>> kvar = K.ones((3,4)) >>> K.eval(kvar) array([[ 1., 1., 1., 1.], [ 1., 1., 1., 1.], [ 1., 1., 1., 1.]], dtype=float32) ``` """ if dtype is None: dtype = tf.float32 Loading @@ -196,10 +167,12 @@ def ones(shape, dtype=None, name=None): def cast_to_floatx(x): """Cast a Numpy array to the default Keras float type. # Arguments Parameters ---------- x: Numpy array. # Returns Returns ------- The same Numpy array, cast to its new type. """ return np.asarray(x, dtype=tf.float32) Loading Loading @@ -227,19 +200,6 @@ def int_shape(x): shape = x.get_shape() return tuple([i.__int__() for i in shape]) def get_value(x): """Returns the value of a variable. Parameters ---------- x: input variable. Returns ------- A Numpy array. """ return x.eval(session=get_session()) def get_uid(prefix=''): """Provides a unique UID given a string prefix. Loading @@ -254,59 +214,6 @@ def get_uid(prefix=''): _UID_PREFIXES[prefix] += 1 return _UID_PREFIXES[prefix] def _initialize_variables(): if hasattr(tf, 'global_variables'): variables = tf.global_variables() else: variables = tf.all_variables() uninitialized_variables = [] for v in variables: if not hasattr(v, '_keras_initialized') or not v._keras_initialized: uninitialized_variables.append(v) v._keras_initialized = True if uninitialized_variables: sess = get_session() if hasattr(tf, 'variables_initializer'): sess.run(tf.variables_initializer(uninitialized_variables)) else: sess.run(tf.initialize_variables(uninitialized_variables)) # TODO(rbharath): DANGEROUS! THIS IS LEAKY AND BREAKS DEEPCHEM STYLE! def get_session(): """Returns the TF session to be used by the backend. If a default TensorFlow session is available, we will return it. Else, we will return the global Keras session. If no global Keras session exists at this point: we will create a new global session. Note that you can manually set the global session via `K.set_session(sess)`. Returns ------- A TensorFlow session. """ global _SESSION if tf.get_default_session() is not None: session = tf.get_default_session() else: if _SESSION is None: if not os.environ.get('OMP_NUM_THREADS'): config = tf.ConfigProto(allow_soft_placement=True) else: nb_thread = int(os.environ.get('OMP_NUM_THREADS')) config = tf.ConfigProto(intra_op_parallelism_threads=nb_thread, allow_soft_placement=True) _SESSION = tf.Session(config=config) session = _SESSION if not _MANUAL_VAR_INIT: _initialize_variables() return session def concatenate(tensors, axis=-1): """Concatenates a list of tensors alongside the specified axis. Loading
deepchem/nn/regularizers.py +0 −35 Original line number Diff line number Diff line Loading @@ -17,41 +17,6 @@ class Regularizer(object): def __call__(self, x): return 0 class EigenvalueRegularizer(Regularizer): """Regularizer based on the eignvalues of a weight matrix. Only available for tensors of rank 2. # Arguments k: Float; modulates the amount of regularization to apply. """ def __init__(self, k): self.k = k def __call__(self, x): if model_ops.get_ndim(x) != 2: raise ValueError('EigenvalueRegularizer ' 'is only available for tensors of rank 2.') covariance = model_ops.dot(tf.transpose(x), x) dim1, dim2 = model_ops.eval(tf.shape(covariance)) # Power method for approximating the dominant eigenvector: power = 9 # Number of iterations of the power method. o = model_ops.ones([dim1, 1]) # Initial values for the dominant eigenvector. main_eigenvect = model_ops.dot(covariance, o) for n in range(power - 1): main_eigenvect = model_ops.dot(covariance, main_eigenvect) covariance_d = model_ops.dot(covariance, main_eigenvect) # The corresponding dominant eigenvalue: main_eigenval = (model_ops.dot(tf.transpose(covariance_d), main_eigenvect) / model_ops.dot(tf.transpose(main_eigenvect), main_eigenvect)) # Multiply by the given regularization gain. regularization = (main_eigenval ** 0.5) * self.k return model_ops.sum(regularization) class L1L2Regularizer(Regularizer): """Regularizer for L1 and L2 regularization. Loading
