close fd's aggressively (1b0608fd) · Commits · github_fork / Clip

fnordmetric-core/src/io/fileutil.cc

+8 −0

Original line number	Diff line number	Diff line
		@@ -12,7 +12,9 @@
		#include <fnordmetric/util/runtimeexception.h>
		#include <fnordmetric/util/stringutil.h>
		#include <string.h>
		#include <sys/fcntl.h>
		#include <sys/stat.h>
		#include <sys/types.h>
		#include <unistd.h>

		namespace fnord {
		@@ -142,5 +144,11 @@ void FileUtil::rm(const std::string& filename) {
		}


		void FileUtil::truncate(const std::string& filename, size_t new_size) {
		if (::truncate(filename.c_str(), new_size) < 0) {
		RAISE_ERRNO(kIOError, "ftruncate(%s) failed", filename.c_str());
		}
		}

		}
		}

fnordmetric-core/src/io/fileutil.h

+5 −0

Original line number	Diff line number	Diff line
		@@ -57,6 +57,11 @@ public:
		*/
		static void rm(const std::string& filename);

		/**
		* Truncate a file
		*/
		static void truncate(const std::string& filename, size_t size);

		};

		}

fnordmetric-core/src/io/mmappedfile.cc

+9 −5

Original line number	Diff line number	Diff line
		@@ -14,8 +14,12 @@
		namespace fnord {
		namespace io {

		MmappedFile::MmappedFile(File&& file) : file_(std::move(file)) {
		size_ = file_.size();
		MmappedFile::MmappedFile(File&& file) {
		File local_file = std::move(file);

		size_ = local_file.size();
		is_writable_ = local_file.isWritable();

		if (size_ == 0) {
		RAISE(kIllegalArgumentError, "can't mmap() empty file");
		}
		@@ -23,9 +27,9 @@ MmappedFile::MmappedFile(File&& file) : file_(std::move(file)) {
		data_ = mmap(
		nullptr,
		size_,
		file_.isWritable() ? PROT_WRITE \| PROT_READ : PROT_READ,
		is_writable_ ? PROT_WRITE \| PROT_READ : PROT_READ,
		MAP_SHARED,
		file_.fd(),
		local_file.fd(),
		0);

		if (data_ == MAP_FAILED) {
		@@ -38,7 +42,7 @@ MmappedFile::~MmappedFile() {
		}

		bool MmappedFile::isWritable() const {
		return file_.isWritable();
		return is_writable_;
		}

		}

fnordmetric-core/src/io/mmappedfile.h

+1 −1

Original line number	Diff line number	Diff line
		@@ -53,7 +53,7 @@ public:
		bool isWritable() const;

		protected:
		File file_;
		bool is_writable_;
		void* data_;
		size_t size_;
		};

fnordmetric-core/src/io/pagemanager.cc

+20 −37

Original line number	Diff line number	Diff line
		@@ -12,6 +12,8 @@
		#include <sys/mman.h>
		#include <unistd.h>
		#include "pagemanager.h"
		#include <fnordmetric/io/file.h>
		#include <fnordmetric/io/fileutil.h>

		namespace fnord {
		namespace io {
		@@ -86,28 +88,19 @@ void* PageManager::PageRef::operator*() const {

		PageManager::PageRef::~PageRef() {};

		MmapPageManager::MmapPageManager(int fd, size_t len, size_t block_size) :
		PageManager(block_size, len),
		fd_(fd),
		file_size_(len),
		MmapPageManager::MmapPageManager(
		const std::string& filename,
		size_t file_size) :
		PageManager(1, file_size),
		filename_(filename),
		file_size_(file_size),
		current_mapping_(nullptr) {}

		//MmapPageManager::MmapPageManager(
		// int fd,
		// size_t len,
		// size_t block_size,
		// const LogSnapshot& log_snapshot) :
		// PageManager(block_size, log_snapshot),
		// fd_(fd),
		// file_size_(len),
		// current_mapping_(nullptr) {}

		MmapPageManager::MmapPageManager(MmapPageManager&& move) :
		PageManager(std::move(move)),
		fd_(move.fd_),
		filename_(move.filename_),
		file_size_(move.file_size_),
		current_mapping_(move.current_mapping_) {
		move.fd_ = -1;
		move.file_size_ = 0;
		move.current_mapping_ = nullptr;
		}
		@@ -116,8 +109,6 @@ MmapPageManager::~MmapPageManager() {
		if (current_mapping_ != nullptr) {
		current_mapping_->decrRefs();
		}

		close(fd_);
		}

		std::unique_ptr<PageManager::PageRef> MmapPageManager::getPage(
		@@ -127,7 +118,8 @@ std::unique_ptr<PageManager::PageRef> MmapPageManager::getPage(
		mmap_mutex_.lock();

		if (last_byte > file_size_) {
		ftruncate(fd_, last_byte); // FIXPAUL check errors
		// FIXPAUL truncate in larger blocks
		FileUtil::truncate(filename_, last_byte);
		file_size_ = last_byte;
		}

		@@ -141,34 +133,29 @@ MmapPageManager::MmappedFile* MmapPageManager::getMmappedFile(
		uint64_t last_byte) {
		if (current_mapping_ == nullptr \|\| last_byte > current_mapping_->size) {
		/* align mmap size to the next larger block boundary */
		uint64_t mmap_size =
		((last_byte + kMmapSizeMultiplier - 1) / kMmapSizeMultiplier) *
		kMmapSizeMultiplier;
		auto file = fnord::io::File::openFile(
		filename_,
		File::O_READ \| File::O_WRITE);

		int fd = dup(fd_);
		if (fd < 0) {
		perror("dup() failed");
		abort(); // FIXPAUL
		}
		auto mmap_size = file.size();

		void* addr = mmap(
		nullptr,
		mmap_size,
		PROT_WRITE \| PROT_READ,
		MAP_SHARED,
		fd,
		file.fd(),
		0);

		if (addr == MAP_FAILED) {
		perror("mmap() failed");
		abort(); // FIXPAUL
		RAISE(kMallocError, "mmap() failed");
		}

		if (current_mapping_ != nullptr) {
		current_mapping_->decrRefs();
		}

		current_mapping_ = new MmappedFile(addr, mmap_size, fd);
		current_mapping_ = new MmappedFile(addr, mmap_size);
		}

		return current_mapping_;
		@@ -176,11 +163,9 @@ MmapPageManager::MmappedFile* MmapPageManager::getMmappedFile(

		MmapPageManager::MmappedFile::MmappedFile(
		void* __data,
		const size_t __size,
		const int __fd) :
		const size_t __size) :
		data(__data),
		size(__size),
		fd(__fd),
		refs(1) {}

		MmapPageManager::MmappedFile::~MmappedFile() {
		@@ -194,8 +179,6 @@ void MmapPageManager::MmappedFile::incrRefs() {
		void MmapPageManager::MmappedFile::decrRefs() {
		if (refs.fetch_sub(1) == 1) {
		assert(refs.load() == 0);
		munmap(data, size);
		close(fd);
		delete this;
		}
		}

Admin message