CSVTableRef read rows (68ccd4a7) · Commits · github_fork / Clip

build/cmake/CMakeLists.txt

+1 −2

Original line number	Diff line number	Diff line
		@@ -58,10 +58,9 @@ add_executable(tests/test-query

		add_executable(tests/test-input-stream
		${FNORDMETRIC_SOURCES}
		${FNORDMETRIC_CSV_BACKEND_SOURCES}
		stage/src/fnordmetric/util/inputstream_test.cc)

		add_executable(tests/test-csv-backend
		${FNORDMETRIC_UI_SOURCES}
		${FNORDMETRIC_QUERY_SOURCES}
		${FNORDMETRIC_CSV_BACKEND_SOURCES}
		stage/src/fnordmetric/query/backends/csv/csvbackend_test.cc)

src/query/backends/csv/csvbackend_test.cc

+21 −5

Original line number	Diff line number	Diff line
		@@ -7,11 +7,11 @@
		#include <stdlib.h>
		#include <stdio.h>
		#include <string.h>
		#include "../../../util/runtimeexception.h"
		#include "../../../util/unittest.h"
		#include "csvinputstream.h"
		#include "csvtableref.h"
		//#include "csv_backend.h"
		#include <fnordmetric/util/runtimeexception.h>
		#include <fnordmetric/util/unittest.h>
		#include <fnordmetric/query/backends/csv/csvinputstream.h>
		#include <fnordmetric/query/backends/csv/csvtableref.h>
		#include <fnordmetric/query/svalue.h>

		using namespace fnordmetric::query::csv_backend;

		@@ -95,3 +95,19 @@ TEST_CASE(CSVInputStreamTest, TestGetColumnIndexWithHeaders, [] () {
		EXPECT_EQ(table_ref.getColumnIndex("country"), 0);
		});

		TEST_CASE(CSVInputStreamTest, TestReadRowsWithHeaders, [] () {
		CSVTableRef table_ref(
		CSVInputStream::openFile("test/fixtures/gbp_per_country_simple.csv"),
		true);

		int num_rows = 0;
		for (;; ++num_rows) {
		std::vector<fnordmetric::query::SValue> target;
		if (!table_ref.readNextRow(&target)) {
		break;
		}
		}

		EXPECT_EQ(num_rows, 191)
		});

src/query/backends/csv/csvtableref.cc

+50 −1

Original line number	Diff line number	Diff line
		@@ -7,7 +7,9 @@
		#include <memory>
		#include <fnordmetric/query/backends/csv/csvtableref.h>
		#include <fnordmetric/query/backends/csv/csvinputstream.h>
		#include <fnordmetric/query/svalue.h>
		#include <fnordmetric/query/tableref.h>
		#include <fnordmetric/query/tablescan.h>
		#include <fnordmetric/util/runtimeexception.h>

		namespace fnordmetric {
		@@ -18,7 +20,9 @@ CSVTableRef::CSVTableRef(
		std::unique_ptr<CSVInputStream>&& csv,
		bool headers /* = false */) :
		csv_(std::move(csv)),
		num_cols_(-1) {
		num_cols_(-1),
		min_cols_(0),
		row_index_(0) {
		if (headers) {
		readHeaders();
		}
		@@ -35,6 +39,49 @@ int CSVTableRef::getColumnIndex(const std::string& name) {
		}

		void CSVTableRef::executeScan(TableScan* scan) {
		for (;;) {
		std::vector<SValue> row;

		if (!readNextRow(&row)) {
		return;
		}

		if (!scan->nextRow(row.data(), row.size())) {
		return;
		}
		}
		}

		bool CSVTableRef::readNextRow(std::vector<SValue>* target) {
		std::vector<std::string> row;

		if (!csv_->readNextRow(&row) \|\| row.size() == 0) {
		return false;
		}

		if (row.size() < min_cols_) {
		RAISE(
		util::RuntimeException,
		"csv row #%i does not have enough columns -- "
		"columns found=%i, required=%i\n",
		row_index_,
		row.size(),
		min_cols_); // FIXPAUL filename
		}

		if (num_cols_ == -1) {
		num_cols_ = row.size();
		} else if (row.size() != num_cols_) {
		RAISE(
		util::RuntimeException,
		"csv row #%i does not have the same number of columns as the previous "
		"line -- number of columns found=%i, previous=%i\n",
		row_index_,
		row.size(),
		num_cols_); // FIXPAUL filename
		}

		return true;
		}

		void CSVTableRef::readHeaders() {
		@@ -50,6 +97,8 @@ void CSVTableRef::readHeaders() {
		}

		num_cols_ = col_index;
		min_cols_ = col_index;
		row_index_++;
		}

		}

src/query/backends/csv/csvtableref.h

+5 −2

Original line number	Diff line number	Diff line
		@@ -14,6 +14,7 @@

		namespace fnordmetric {
		namespace query {
		class SValue;
		namespace csv_backend {

		/**
		@@ -29,13 +30,15 @@ public:
		int getColumnIndex(const std::string& name) override;
		void executeScan(TableScan* scan) override;

		protected:

		bool readNextRow(std::vector<SValue>* target);
		void readHeaders();

		protected:
		std::unordered_map<std::string, size_t> headers_;
		std::unique_ptr<CSVInputStream> csv_;
		int num_cols_;
		int min_cols_;
		int row_index_;
		};

		}

src/query/svalue.cc

+4 −4

Original line number	Diff line number	Diff line
		@@ -21,6 +21,10 @@ SValue::SValue() {
		data_.type = T_UNDEFINED;
		}

		SValue::~SValue() {
		// FIXPAUL free string!
		}

		SValue::SValue(const std::string& string_value) {
		data_.type = T_STRING;
		data_.u.t_string.len = string_value.size();
		@@ -112,10 +116,6 @@ bool SValue::operator==(const SValue& other) const {
		}
		}

		SValue::~SValue() {
		// FIXPAUL free string!
		}

		SValue::kSValueType SValue::getType() const {
		return data_.type;
		}

Admin message