remove trailing zero bytes (padding) when finalizing sstables (f6617c91) · Commits · github_fork / Clip

fnordmetric-core/src/io/pagemanager.cc

+29 −2

Original line number	Diff line number	Diff line
		@@ -95,6 +95,7 @@ MmapPageManager::MmapPageManager(
		PageManager(1, file_size),
		filename_(filename),
		file_size_(file_size),
		used_bytes_(file_size),
		current_mapping_(nullptr) {
		sys_page_size_ = sysconf(_SC_PAGESIZE);
		}
		@@ -103,9 +104,11 @@ MmapPageManager::MmapPageManager(MmapPageManager&& move) :
		PageManager(std::move(move)),
		filename_(move.filename_),
		file_size_(move.file_size_),
		used_bytes_(move.used_bytes_),
		current_mapping_(move.current_mapping_),
		sys_page_size_(move.sys_page_size_) {
		move.file_size_ = 0;
		move.used_bytes_ = 0;
		move.current_mapping_ = nullptr;
		}

		@@ -115,15 +118,39 @@ MmapPageManager::~MmapPageManager() {
		}
		}

		void MmapPageManager::shrinkFile() {
		FileUtil::truncate(filename_, used_bytes_);
		}

		std::unique_ptr<PageManager::PageRef> MmapPageManager::getPage(
		const PageManager::Page& page) {
		return getPageImpl(page, true);
		}

		std::unique_ptr<PageManager::PageRef> MmapPageManager::getPage(
		const PageManager::Page& page,
		kNoPadding no_padding) {
		return getPageImpl(page, false);
		}

		std::unique_ptr<PageManager::PageRef> MmapPageManager::getPageImpl(
		const PageManager::Page& page,
		bool allow_padding) {
		uint64_t last_byte = page.offset + page.size;

		mmap_mutex_.lock();

		if (last_byte > used_bytes_) {
		used_bytes_ = last_byte;
		}

		if (last_byte > file_size_) {
		auto new_size =
		((last_byte / kMmapSizeMultiplier) + 1) * kMmapSizeMultiplier;
		size_t new_size;
		if (allow_padding) {
		new_size = ((last_byte / kMmapSizeMultiplier) + 1) * kMmapSizeMultiplier;
		} else {
		new_size = last_byte;
		}

		if (fnordmetric::env()->verbose()) {
		fnordmetric::env()->logger()->printf(

fnordmetric-core/src/io/pagemanager.h

+17 −0

Original line number	Diff line number	Diff line
		@@ -163,8 +163,24 @@ public:
		std::unique_ptr<PageManager::PageRef> getPage(
		const PageManager::Page& page) override;

		/**
		* Request a page to be mapped into memory but disallow padding the file
		* . Returns a smart pointer.
		*/
		struct kNoPadding {};
		std::unique_ptr<PageManager::PageRef> getPage(
		const PageManager::Page& page, kNoPadding no_padding);


		/**
		* Truncate the file to the actual used size
		*/
		void shrinkFile();

		protected:

		std::unique_ptr<PageManager::PageRef> getPageImpl(
		const PageManager::Page& page, bool allow_padding);
		/**
		* Returns a mmap()ed memory region backend by the managed file spans until
		* at least last_byte
		@@ -172,6 +188,7 @@ protected:
		MmappedFile* getMmappedFile(uint64_t last_byte);

		const std::string filename_;
		size_t used_bytes_;
		size_t file_size_;
		MmappedFile* current_mapping_;
		std::mutex mmap_mutex_;

fnordmetric-core/src/sstable/sstablewriter.cc

+3 −1

Original line number	Diff line number	Diff line
		@@ -143,7 +143,7 @@ void SSTableWriter::writeIndex(uint32_t index_type, void* data, size_t size) {
		}

		auto alloc = mmap_->allocPage(sizeof(BinaryFormat::FooterHeader) + size);
		auto page = mmap_->getPage(alloc);
		auto page = mmap_->getPage(alloc, io::MmapPageManager::kNoPadding{});

		auto header = page->structAt<BinaryFormat::FooterHeader>(0);
		header->magic = BinaryFormat::kMagicBytes;
		@@ -159,6 +159,7 @@ void SSTableWriter::writeIndex(uint32_t index_type, void* data, size_t size) {
		}

		page->sync();
		mmap_->shrinkFile();
		}

		void SSTableWriter::reopen(size_t file_size) {
		@@ -193,6 +194,7 @@ void SSTableWriter::finalize() {
		header.updateBodySize(body_size_);

		page->sync();
		mmap_->shrinkFile();
		}

		// FIXPAUL lock

Admin message