Commit d5fd17a5 authored by Peter Eastman's avatar Peter Eastman
Browse files

Possible fix for error on travis

parent 0890181b
Loading
Loading
Loading
Loading
+4 −2
Original line number Diff line number Diff line
@@ -167,8 +167,10 @@ class DataLoader(object):
        worker_pool = mp.Pool(processes=1)
    log("Spawning workers now.", self.verbosity)
    metadata_rows = []
    data_iterator = zip(
        it.repeat((self, shard_size, input_type, data_dir)),
    def wrap_with_shard_metadata(iterator):
      for item in iterator:
        yield ((self, shard_size, input_type, data_dir), item)
    data_iterator = wrap_with_shard_metadata(
        enumerate(load_data(input_files, shard_size, self.verbosity)))
    # Turns out python map is terrible and exhausts the generator as given.
    # Solution seems to be to to manually pull out N elements from iterator,