Unverified Commit acc18463 authored by Bharath Ramsundar's avatar Bharath Ramsundar Committed by GitHub
Browse files

Merge pull request #1013 from lilleswing/manifest

Add manifest to exclude datafiles
parents 79df7c75 a1bda3fd
Loading
Loading
Loading
Loading

MANIFEST.in

0 → 100644
+2 −0
Original line number Diff line number Diff line
prune datasets
prune examples
+10 −8
Original line number Diff line number Diff line
@@ -624,7 +624,8 @@ class DiskDataset(Dataset):
    if not len(self.metadata_df):
      raise ValueError("No data in dataset.")
    sample_X = load_from_disk(
        os.path.join(self.data_dir, next(self.metadata_df.iterrows())[1]['X']))
        os.path.join(self.data_dir,
                     next(self.metadata_df.iterrows())[1]['X']))
    return np.shape(sample_X)[1:]

  def get_shard_size(self):
@@ -632,7 +633,8 @@ class DiskDataset(Dataset):
    if not len(self.metadata_df):
      raise ValueError("No data in dataset.")
    sample_y = load_from_disk(
        os.path.join(self.data_dir, next(self.metadata_df.iterrows())[1]['y']))
        os.path.join(self.data_dir,
                     next(self.metadata_df.iterrows())[1]['y']))
    return len(sample_y)

  def _get_metadata_filename(self):