Debugging singletask models.

    """

    if not len(self.metadata_df):

      raise ValueError("No data in dataset.")

    return self.metadata_df.iterrows().next()[1]['task_names']

    return sorted(self.metadata_df.iterrows().next()[1]['task_names'])

  def get_data_shape(self):

    """

    dangerous (!) for large datasets which don't fit into memory.

    """

    Xs, ys, ws, ids = [], [], [], []

    for (X_b, y_b, w_b, ids_b) in self.iterbatches():

    for (X_b, y_b, w_b, ids_b) in self.itershards():

      Xs.append(X_b)

      ys.append(y_b)

      ws.append(w_b)

      ids.append(np.squeeze(ids_b))

    # ids_b should be 1-d. Squeeze to make sure

    return (np.vstack(Xs), np.vstack(ys), np.vstack(ws),

            np.squeeze(np.vstack(ids)))

            np.concatenate(ids))

  def get_labels(self):

    """

  # TODO(rbharath): This is a hack. clean up.

  if not len(df):

    return None

  task_names = sorted(tasks)

  ids, X, y, w = _df_to_numpy(df, feature_types, tasks)

  sorted_tasks = sorted(tasks)

  ids, X, y, w = _df_to_numpy(df, feature_types, sorted_tasks)

  X_sums, X_sum_squares, X_n = compute_sums_and_nb_sample(X)

  y_sums, y_sum_squares, y_n = compute_sums_and_nb_sample(y, w)

  save_to_disk(ids, out_ids)

  # TODO(rbharath): Should X be saved to out_X_transformed as well? Since

  # itershards expects to loop over X-transformed? (Ditto for y/w)

  return([df_file, task_names, out_ids, out_X, out_X_transformed, out_y,

  return([df_file, sorted_tasks, out_ids, out_X, out_X_transformed, out_y,

          out_y_transformed, out_w, out_w_transformed,

          out_X_sums, out_X_sum_squares, out_X_n,

          out_y_sums, out_y_sum_squares, out_y_n])

# TODO(rbharath): This function is complicated enough that it should have unit

# tests.

def _df_to_numpy(df, feature_types, tasks):

  """Transforms a featurized dataset df into standard set of numpy arrays"""

  if not set(feature_types).issubset(df.keys()):

    feature_list = []

    for feature_type in feature_types:

      feature_list.append(datapoint[feature_type])

    # TODO(rbharath): Total hack. Fix before merge!!!

    try:

      features = np.squeeze(np.concatenate(feature_list))

      for feature_ind, val in enumerate(features):

      raise ValueError("tasks must be a list.")

    assert verbosity in [None, "low", "high"]

    self.verbosity = verbosity

    self.tasks = tasks

    self.sorted_tasks = sorted(tasks)

    self.smiles_field = smiles_field

    self.split_field = split_field

    if id_field is None:

    else:

      raise ValueError("Unrecognized input_type")

    if self.threshold is not None:

      for task in self.tasks:

      for task in self.sorted_tasks:

        raw = _process_field(data[task])

        if not isinstance(raw, float):

          raise ValueError("Cannot threshold non-float fields.")

    df = pd.DataFrame(ori_df[[self.id_field]])

    df.columns = ["mol_id"]

    df["smiles"] = ori_df[[self.smiles_field]]

    for task in self.tasks:

    for task in self.sorted_tasks:

      df[task] = ori_df[[task]]

    if self.user_specified_features is not None:

      for feature in self.user_specified_features:

    self.dataset_files = load_from_disk(self._get_dataset_paths_filename())

    if os.path.exists(self._get_compounds_filename()) and reload:

      log("Loading prexisting compounds from disk", self.verbosity)

      log("Loading prexisting compounds from disk", self.verbosity, "high")

      compounds_df = load_from_disk(self._get_compounds_filename())

    else:

      log("Saving compounds to disk", self.verbosity)

      log("Saving compounds to disk", self.verbosity, "high")

      compounds_df = self._get_compounds()

      # compounds_df is not altered by any method after initialization, so it's

      # safe to keep a copy in memory and on disk.

    # Create empty dataset

    data_dir = tempfile.mkdtemp() 

    featurizers = None

    tasks = self.task_types.keys()

    tasks = sorted(self.task_types.keys())

    batch_dataset = Dataset(data_dir=data_dir, samples=None,

                            featurizers=featurizers, tasks=tasks,

                            use_user_specified_features=True)

    return batch_dataset

  # TODO(rbharath): The structure of the produced df might be

  # complicated. Better way to model?

  def predict(self, dataset, transformers):

    """

    Uses self to make predictions on provided Dataset object.

    """

    task_names = dataset.get_task_names()

    task_names = sorted(dataset.get_task_names())

    pred_task_names = ["%s_pred" % task_name for task_name in task_names]

    w_task_names = ["%s_weight" % task_name for task_name in task_names]

    raw_task_names = [task_name+"_raw" for task_name in task_names]

                    + ["y_means", "y_stds"])

    pred_y_df = pd.DataFrame(columns=column_names)

    batch_size = self.model_params["batch_size"]

    for (X_batch, y_batch, w_batch, ids_batch) in dataset.iterbatches(batch_size):

    #batch_size = self.model_params["batch_size"]

    #for (X_batch, y_batch, w_batch, ids_batch) in dataset.iterbatches(batch_size):

    for (X_batch, y_batch, w_batch, ids_batch) in dataset.itershards():

      # HACK(JG): This was a hack to perform n-fold averaging of y_pred on

      # a given X_batch.  If fit_transformers exist, we will apply them to

  def __init__(self, task_types, model_params, model_dir, model_builder,

               verbosity=None):

    self.task_types = task_types

    self.sorted_tasks = sorted(self.task_types.keys())

    self.model_params = model_params

    self.models = {}

    self.model_dir = model_dir

    Warning: This current implementation is only functional for sklearn models. 

    """

    X, y, _, _ = dataset.to_numpy()

    for ind, task in enumerate(self.task_types.keys()):

    for ind, task in enumerate(self.sorted_tasks):

      log("Fitting model for task %s" % task, self.verbosity, "high")

      y_task = y[:, ind]

      self.models[task].raw_model.fit(X, y_task)

    """

    Concatenates results from all singletask models.

    """

    N_tasks = len(self.task_types.keys())

    N_tasks = len(self.sorted_tasks)

    N_samples = X.shape[0]

    y_pred = np.zeros((N_samples, N_tasks))

    for ind, task in enumerate(self.task_types.keys()):

    for ind, task in enumerate(self.sorted_tasks):

      y_pred[:, ind] = self.models[task].predict_on_batch(X)

    return y_pred

#  def predict(self, dataset, transformers):

#    """

#    Generate predictions of models on dataset.

#    """

#    task_names = sorted(dataset.get_task_names())

#    pred_y_df = self._create_prediction_datafram(dataset) 

#

  def save(self):

    """Save all models"""

    for task in self.task_types.keys():

    for task in self.sorted_tasks:

      log("Saving model for task %s" % task, self.verbosity, "high")

      self.models[task].save()

  def load(self):

    """Load all models"""

    for task in self.task_types.keys():

    for task in self.sorted_tasks:

      log("Loading model for task %s" % task, self.verbosity, "high")

      self.models[task].load()