#!/usr/bin/python

import sys
import iopro
import unittest
from generate import generate_dataset, IntIter, MissingValuesIter, FixedWidthIter
import StringIO
import numpy
from numpy.testing import assert_array_equal
import gzip
import os


class TestTextAdapter(unittest.TestCase):

    num_records = 100000

    def assert_equality(self, left, right):
        try:
            self.assertTrue(left == right)
        except AssertionError:
            raise AssertionError('FAIL: {0} != {1}'.format(left, right))

    # Basic parsing tests
    def test_string_parsing(self):
        data = StringIO.StringIO('1,2,3\n')
        adapter = iopro.CSVTextAdapter(data, field_names=False, infer_types=False, delimiter=',')
        adapter.set_field_types({0:'S5', 1:'S5', 2:'S5'})
        data = StringIO.StringIO('1,2,3\n')

    def test_no_whitespace_stripping(self):
        data = StringIO.StringIO('1  ,2  ,3  \n')
        adapter = iopro.CSVTextAdapter(data, field_names=False, infer_types=False, delimiter=',')
        adapter.set_field_types({0:'S3', 1:'S3', 2:'S3'})
        self.assert_equality(adapter[0].item(), ("1  ", "2  ", "3  "))

        data = StringIO.StringIO('  1,  2,  3\n')
        adapter = iopro.CSVTextAdapter(data, field_names=False, infer_types=False, delimiter=',')
        adapter.set_field_types({0:'S3', 1:'S3', 2:'S3'})
        self.assert_equality(adapter[0].item(), ("  1", "  2", "  3"))

        data = StringIO.StringIO('  1  ,  2  ,  3  \n')
        adapter = iopro.CSVTextAdapter(data, field_names=False, infer_types=False, delimiter=',')
        adapter.set_field_types({0:'S5', 1:'S5', 2:'S5'})
        self.assert_equality(adapter[0].item(), ("  1  ", "  2  ", "  3  "))

        data = StringIO.StringIO('\t1\t,\t2\t,\t3\t\n')
        adapter = iopro.CSVTextAdapter(data, field_names=False, infer_types=False, delimiter=',')
        adapter.set_field_types({0:'S3', 1:'S3', 2:'S3'})
        self.assert_equality(adapter[0].item(), ("\t1\t", "\t2\t", "\t3\t"))

    def test_quoted_whitespace(self):
        data = StringIO.StringIO('"1  ","2  ","3  "\n')
        adapter = iopro.CSVTextAdapter(data, field_names=False, infer_types=False, delimiter=',')
        adapter.set_field_types({0:'S3', 1:'S3', 2:'S3'})
        self.assert_equality(adapter[0].item(), ("1  ", "2  ", "3  "))

        data = StringIO.StringIO('"\t1\t"\t"\t2\t"\t"\t3\t"\n')
        adapter = iopro.CSVTextAdapter(data, field_names=False, infer_types=False, delimiter='\t')
        adapter.set_field_types({0:'S3', 1:'S3', 2:'S3'})
        self.assert_equality(adapter[0].item(), ("\t1\t", "\t2\t", "\t3\t"))

    def test_fixed_simple(self):
        # TODO: fix this test on 32-bit and on Windows
        if tuple.__itemsize__ == 4:
            # This test does not work on 32-bit, so we skip it
            return
        if sys.platform == 'win32':
            # This test does not work on Windows
            return
        data = StringIO.StringIO("  1  2  3\n  4  5 67\n890123  4")
        adapter = iopro.FixedWidthTextAdapter(data, 3, infer_types=False, field_names=False)
        adapter.set_field_types({0:'i', 1:'i', 2:'i'})

        control = [(1, 2, 3), (4, 5, 67), (890, 123, 4)]
        self.assert_(control == adapter[:].tolist())

    def test_spaces_around_numeric_values(self):
        data = StringIO.StringIO(' 1 , -2 , 3.3 , -4.4 \n  5  ,  -6  ,  7.7 , -8.8 ')
        adapter = iopro.CSVTextAdapter(data, field_names=False, infer_types=False)
        adapter.set_field_types({0:'u4', 1:'i8', 2:'f4', 3:'f8'})
        array = adapter[:]

        assert_array_equal(array, numpy.array([(1,-2,3.3,-4.4),
                                               (5,-6,7.7,-8.8)],
                                               dtype='u4,i8,f4,f8'))


    def test_slicing(self):

        data = StringIO.StringIO()
        generate_dataset(data, IntIter(), ',', self.num_records)
        adapter = iopro.CSVTextAdapter(data, delimiter=',', field_names=False, infer_types=False)
        adapter.set_field_types({0:'u4',1:'u4',2:'u4',3:'u4',4:'u4'})

        self.assert_equality(adapter[0].item(), (0, 1, 2, 3, 4))
        expected_values = [((self.num_records-1)*5)+x for x in range(5)]
        self.assert_equality(adapter[self.num_records-1].item(), tuple(expected_values))

        #adapter.create_index()
        #self.assert_equality(adapter[-1].item(), tuple(expected_values))

        self.assert_equality(adapter['f0'][0].item(), (0,))
        self.assert_equality(adapter['f4'][1].item(), (9,))
        #self.assert_equality(adapter[self.num_records-1]['f4'], (self.num_records*5)-1)
 
        array = adapter[:]
        record = [x for x in range(0, 5)]
        self.assert_equality(array.size, self.num_records)
        for i in range(0, self.num_records):
            self.assert_equality(array[i].item(), tuple(record))
            record = [x+5 for x in record]

        array = adapter[:-1]
        record = [x for x in range(0, 5)]
        self.assert_equality(array.size, self.num_records-1)
        for i in range(0, self.num_records-1):
            self.assert_equality(array[i].item(), tuple(record))
            record = [x+5 for x in record]

        array = adapter[0:10]
        self.assert_equality(array.size, 10)
        record = [x for x in range(0, 5)]
        for i in range(0, 10):
            self.assert_equality(array[i].item(), tuple(record))
            record = [x+5 for x in record]

        array = adapter[1:]
        self.assert_equality(array.size, self.num_records-1)
        record = [x for x in range(5, 10)]
        for i in range(0, self.num_records-1):
            self.assert_equality(array[i].item(), tuple(record))
            record = [x+5 for x in record]

        array = adapter[0:10:2]
        self.assert_equality(array.size, 5)
        record = [x for x in range(0, 5)]
        for i in range(0, 5):
            self.assert_equality(array[i].item(), tuple(record))
            record = [x+10 for x in record]

        array = adapter[['f0', 'f4']][:]
        record = [0, 4]
        self.assert_equality(array.size, self.num_records)
        for i in range(0, self.num_records):
            self.assert_equality(array[i].item(), tuple(record))
            record = [x+5 for x in record]

        adapter.field_filter = [0, 'f4']
        array = adapter[:]
        record = [0, 4]
        self.assert_equality(array.size, self.num_records)
        for i in range(0, self.num_records):
            self.assert_equality(array[i].item(), tuple(record))
            record = [x+5 for x in record]

        adapter.field_filter = None
        array = adapter[:]
        record = [0, 1, 2, 3, 4]
        self.assert_equality(array.size, self.num_records)
        for i in range(0, self.num_records):
            self.assert_equality(array[i].item(), tuple(record))
            record = [x+5 for x in record]

        try:
            adapter[self.num_records]
        except iopro.AdapterIndexError:
            pass
        else:
            self.fail('AdaperIndexError not thrown')

        try:
            adapter[0:self.num_records+1]
        except iopro.AdapterIndexError:
            pass
        else:
            self.fail('AdaperIndexError not thrown')


    def test_converters(self):

        data = StringIO.StringIO()
        generate_dataset(data, IntIter(), ',', self.num_records)
        adapter = iopro.CSVTextAdapter(data, delimiter=',', field_names=False)
        #adapter.set_field_types({0:'u4', 1:'u4', 2:'u4', 3:'u4', 4:'u4'})

        def increment(input_str):
            return int(input_str) + 1

        def double(input_str):
            return int(input_str) + int(input_str)

        if sys.platform == 'win32' and tuple.__itemsize__ == 8:
            # TODO: there problems below here 64-bit Windows, I get
            # OverflowError: can't convert negative value to unigned PY_LONG_LONG
            return

        adapter.set_converter(0, increment)
        adapter.set_converter('f1', double)

        array = adapter[:]

        self.assert_equality(array.size, self.num_records)
        
        record = [1, 2, 2, 3, 4]
        for i in range(0, self.num_records):
            self.assert_equality(array[i].item(), tuple(record))
            record[0] += 5
            record[1] = (10 * (i+1)) + 2
            record[2] += 5
            record[3] += 5
            record[4] += 5


    def test_missing_fill_values(self):

        data = StringIO.StringIO()
        generate_dataset(data, MissingValuesIter(), ',', self.num_records)

        adapter = iopro.CSVTextAdapter(data, delimiter=',', field_names=False, infer_types=False)
        adapter.set_field_types({'f0':'u4', 1:'u4', 2:'u4', 3:'u4', 'f4':'u4'})
        adapter.set_missing_values({0:['NA', 'NaN'], 'f4':['xx','inf']})
        adapter.set_fill_values({0:99, 4:999})

        array = adapter[:]

        self.assert_equality(array.size, self.num_records)
        
        record = [x for x in range(0, 5)]
        for i in range(0, self.num_records):
            if i % 4 == 0 or i % 4 == 1:
                record[0] = 99
                record[4] = 999
            else:
                record[0] = record[1] - 1
                record[4] = record[3] + 1
            self.assert_equality(array[i].item(), tuple(record))
            record[1] += 5
            record[2] += 5
            record[3] += 5

        adapter = iopro.CSVTextAdapter(data, delimiter=',', field_names=False, infer_types=True)
        adapter.set_missing_values({0:['NA', 'NaN'], 4:['xx','inf']})

        array = adapter[:]

        self.assert_equality(array.size, self.num_records)
        
        record = [x for x in range(0, 5)]
        for i in range(0, self.num_records):
            if i % 4 == 0 or i % 4 == 1:
                record[0] = 0 
                record[4] = 0
            else:
                record[0] = record[1] - 1
                record[4] = record[3] + 1
            self.assert_equality(array[i].item(), tuple(record))
            record[1] += 5
            record[2] += 5
            record[3] += 5

        # Test missing field
        data = StringIO.StringIO('1,2,3\n4,5\n7,8,9')
        adapter = iopro.text_adapter(data, field_names=False)
        adapter.field_types = {0:'O', 1:'O', 2:'O'}
        adapter.set_fill_values({0:numpy.nan, 1:numpy.nan, 2:numpy.nan})
        array = adapter[:]
        assert_array_equal(array, numpy.array([('1','2','3'),
                                               ('4','5',numpy.nan),
                                               ('7','8','9')],
                                               dtype='O,O,O'))


    def test_fixed_width(self):

        data = StringIO.StringIO()
        generate_dataset(data, FixedWidthIter(), '', self.num_records)
        adapter = iopro.FixedWidthTextAdapter(data, [2,3,4,5,6], field_names=False, infer_types=False)
        adapter.set_field_types({0:'u4',1:'u4',2:'u4',3:'u4',4:'u4'})

        array = adapter[:]

        self.assert_equality(array.size, self.num_records)
        
        record = [0, 0, 0, 0, 0]
        for i in range(0, self.num_records):
            self.assert_equality(array[i].item(), tuple(record))
            record = [x+1 for x in record]
            if record[0] == 100:
                record[0] = 0
            if record[1] == 1000:
                record[1] = 0
            if record[2] == 10000:
                record[2] = 0
            if record[3] == 100000:
                record[3] = 0
            if record[4] == 1000000:
                record[4] = 0

        # Test skipping blank lines
        data = StringIO.StringIO(' 1 2 3\n\n 4 5 6')
        adapter = iopro.text_adapter(data, parser='fixed_width',
            field_widths=[2,2,2], field_names=False)
        array = adapter[:]
        assert_array_equal(array, numpy.array([(1,2,3), (4,5,6)],
            dtype=[('f0','<u8'),('f1','<u8'),('f2','<u8')]))

        # Test comment lines
        data = StringIO.StringIO('# 1 2 3\n 1 2 3\n# foo\n 4 5 6')
        adapter = iopro.text_adapter(data, parser='fixed_width',
            field_widths=[2,2,2], field_names=False)
        array = adapter[:]
        assert_array_equal(array, numpy.array([(1,2,3), (4,5,6)],
            dtype=[('f0','<u8'),('f1','<u8'),('f2','<u8')]))

        # Test field names line
        data = StringIO.StringIO(' a b c\n 1 2 3')
        adapter = iopro.text_adapter(data, parser='fixed_width',
            field_widths=[2,2,2], field_names=True)
        array = adapter[:]
        assert_array_equal(array, numpy.array([(1,2,3)],
            dtype=[('a','<u8'),('b','<u8'),('c','<u8')]))

        # Test field names line as comment line
        data = StringIO.StringIO('# a b c\n 1 2 3')
        adapter = iopro.text_adapter(data, parser='fixed_width',
            field_widths=[2,2,2], field_names=True)
        array = adapter[:]
        assert_array_equal(array, numpy.array([(1,2,3)],
            dtype=[('a','<u8'),('b','<u8'),('c','<u8')]))

        # Test incomplete field names line
        data = StringIO.StringIO(' a\n 1 2 3')
        adapter = iopro.text_adapter(data, parser='fixed_width',
            field_widths=[2,2,2], field_names=True)
        array = adapter[:]
        assert_array_equal(array, numpy.array([(1,2,3)],
            dtype=[('a','<u8'),('f1','<u8'),('f2','<u8')]))

    def test_regex(self):

        data = StringIO.StringIO()
        generate_dataset(data, IntIter(), ',', self.num_records)
        adapter = iopro.RegexTextAdapter(data, '([0-9]*),([0-9]*),([0-9]*),([0-9]*),([0-9]*)\n', field_names=False, infer_types=False)
        adapter.set_field_types({0:'u4',1:'u4',2:'u4',3:'u4',4:'u4'})        

        array = adapter[:]

        self.assert_equality(array.size, self.num_records)
        
        record = [x for x in range(0, 5)]
        for i in range(0, self.num_records):
            self.assert_equality(array[i].item(), tuple(record))
            record = [x+5 for x in record]

        # Test skipping blank lines
        data = StringIO.StringIO('1 2 3\n\n4 5 6')
        adapter = iopro.text_adapter(data, parser='regex',
            regex_string='([0-9]) ([0-9]) ([0-9])', field_names=False)
        array = adapter[:]
        assert_array_equal(array, numpy.array([(1,2,3), (4,5,6)],
            dtype=[('f0','<u8'),('f1','<u8'),('f2','<u8')]))

        # Test comment lines
        data = StringIO.StringIO('#1 2 3\n1 2 3\n# foo\n4 5 6')
        adapter = iopro.text_adapter(data, parser='regex',
            regex_string='([0-9]) ([0-9]) ([0-9])', field_names=False)
        array = adapter[:]
        assert_array_equal(array, numpy.array([(1,2,3), (4,5,6)],
            dtype=[('f0','<u8'),('f1','<u8'),('f2','<u8')]))

        # Test field names line
        data = StringIO.StringIO('a b c\n4 5 6')
        adapter = iopro.text_adapter(data, parser='regex',
            regex_string='([0-9]) ([0-9]) ([0-9])', field_names=True)
        array = adapter[:]
        assert_array_equal(array, numpy.array([(4,5,6)],
            dtype=[('a','<u8'),('b','<u8'),('c','<u8')]))

        # Test field names line as comment line
        data = StringIO.StringIO('#a b c\n4 5 6')
        adapter = iopro.text_adapter(data, parser='regex',
            regex_string='([0-9]) ([0-9]) ([0-9])', field_names=True)
        array = adapter[:]
        assert_array_equal(array, numpy.array([(4,5,6)],
            dtype=[('a','<u8'),('b','<u8'),('c','<u8')]))

        # Test incomplete field names line
        data = StringIO.StringIO('a b\n4 5 6')
        adapter = iopro.text_adapter(data, parser='regex',
            regex_string='([0-9]) ([0-9]) ([0-9])', field_names=True)
        array = adapter[:]
        assert_array_equal(array, numpy.array([(4,5,6)],
            dtype=[('a','<u8'),('b','<u8'),('f2','<u8')]))

        # Test field names line that doesn't match regex
        data = StringIO.StringIO('a b c\n1 2  3 4  5 6')
        adapter = iopro.text_adapter(data, parser='regex',
            regex_string='([0-9\s]+)  ([0-9\s]+)  ([0-9\s]+)', field_names=True)
        array = adapter[:]
        assert_array_equal(array, numpy.array([('1 2', '3 4', '5 6')],
            dtype=[('a','O'),('b','O'),('c','O')]))


    def test_index(self):
        if sys.platform == 'win32':
            # TODO: this test fails on Windows because of file lock problems
            return

        num_records = 100000
        expected_values = [((num_records-1)*5) + x for x in range(5)]

        data = StringIO.StringIO()
        generate_dataset(data, IntIter(), ',', num_records)
        
        # test explicit index building
        adapter = iopro.CSVTextAdapter(data, delimiter=',', field_names=False, infer_types=False)
        adapter.set_field_types({0:'u4',1:'u4',2:'u4',3:'u4',4:'u4'})
        adapter.create_index()

        self.assert_equality(adapter[0].item(), tuple([(0*5) + x for x in range(5)]))
        self.assert_equality(adapter[10].item(), tuple([(10*5) + x for x in range(5)]))
        self.assert_equality(adapter[100].item(), tuple([(100*5) + x for x in range(5)]))
        self.assert_equality(adapter[1000].item(), tuple([(1000*5) + x for x in range(5)]))
        self.assert_equality(adapter[10000].item(), tuple([(10000*5) + x for x in range(5)]))
        self.assert_equality(adapter[num_records - 1].item(), tuple([((num_records - 1)*5) + x for x in range(5)]))
        #self.assert_equality(adapter[-1].item(), tuple(expected_values))

        # test implicitly creating disk index on the fly
        if os.path.exists('test.idx'):
            os.remove('test.idx')
        adapter = iopro.CSVTextAdapter(data, delimiter=',', field_names=False, infer_types=False, index_name='test.idx')
        adapter.set_field_types({0:'u4',1:'u4',2:'u4',3:'u4',4:'u4'})
        adapter.to_array()

        self.assert_equality(adapter[0].item(), tuple([(0*5) + x for x in range(5)]))
        self.assert_equality(adapter[10].item(), tuple([(10*5) + x for x in range(5)]))
        self.assert_equality(adapter[100].item(), tuple([(100*5) + x for x in range(5)]))
        self.assert_equality(adapter[1000].item(), tuple([(1000*5) + x for x in range(5)]))
        self.assert_equality(adapter[10000].item(), tuple([(10000*5) + x for x in range(5)]))
        self.assert_equality(adapter[num_records - 1].item(), tuple([((num_records - 1)*5) + x for x in range(5)]))
        #self.assert_equality(adapter[-1].item(), tuple(expected_values))

        adapter.close()

        # test loading disk index
        adapter2 = iopro.CSVTextAdapter(data, delimiter=',', field_names=False, infer_types=False, index_name='test.idx')
        adapter2.set_field_types({0:'u4',1:'u4',2:'u4',3:'u4',4:'u4'})

        self.assert_equality(adapter2[0].item(), tuple([(0*5) + x for x in range(5)]))
        self.assert_equality(adapter2[10].item(), tuple([(10*5) + x for x in range(5)]))
        self.assert_equality(adapter2[100].item(), tuple([(100*5) + x for x in range(5)]))
        self.assert_equality(adapter2[1000].item(), tuple([(1000*5) + x for x in range(5)]))
        self.assert_equality(adapter2[10000].item(), tuple([(10000*5) + x for x in range(5)]))
        self.assert_equality(adapter2[num_records - 1].item(), tuple([((num_records - 1)*5) + x for x in range(5)]))
        #self.assert_equality(adapter2[-1].item(), tuple(expected_values))

        adapter.close()

        os.remove('test.idx')


    def test_gzip_index(self):

        num_records = 1000000

        data = StringIO.StringIO()
        generate_dataset(data, IntIter(), ',', num_records)

        dataz = StringIO.StringIO()
        gzip_output = gzip.GzipFile(fileobj=dataz, mode='wb')
        gzip_output.write(data.getvalue())
        gzip_output.close()

        # test explicit index building
        adapter = iopro.CSVTextAdapter(dataz, compression='gzip', delimiter=',', field_names=False, infer_types=False)
        adapter.set_field_types({0:'u4',1:'u4',2:'u4',3:'u4',4:'u4'})
        adapter.create_index()
    
        self.assert_equality(adapter[0].item(), tuple([(0*5) + x for x in range(5)]))
        self.assert_equality(adapter[10].item(), tuple([(10*5) + x for x in range(5)]))
        self.assert_equality(adapter[100].item(), tuple([(100*5) + x for x in range(5)]))
        self.assert_equality(adapter[1000].item(), tuple([(1000*5) + x for x in range(5)]))
        self.assert_equality(adapter[10000].item(), tuple([(10000*5) + x for x in range(5)]))
        self.assert_equality(adapter[100000].item(), tuple([(100000*5) + x for x in range(5)]))
        self.assert_equality(adapter[num_records - 1].item(), tuple([((num_records - 1)*5) + x for x in range(5)]))
        #self.assert_equality(adapter[-1].item(), tuple(expected_values))

        # test 'trouble' records that have caused crashes in the past
        self.assert_equality(adapter[290000].item(), tuple([(290000*5) + x for x in range(5)]))
        self.assert_equality(adapter[818000].item(), tuple([(818000*5) + x for x in range(5)]))

        # test implicitly creating disk index on the fly
        # JNB: not implemented yet
        '''adapter = iopro.CSVTextAdapter(dataz, compression='gzip', delimiter=',', field_names=False, infer_types=False, indexing=True, index_filename='test.idx')
        adapter.set_field_types({0:'u4',1:'u4',2:'u4',3:'u4',4:'u4'})
        adapter.to_array()

        self.assert_equality(adapter[0].item(), tuple([(0*5) + x for x in range(5)]))
        self.assert_equality(adapter[10].item(), tuple([(10*5) + x for x in range(5)]))
        self.assert_equality(adapter[100].item(), tuple([(100*5) + x for x in range(5)]))
        self.assert_equality(adapter[1000].item(), tuple([(1000*5) + x for x in range(5)]))
        self.assert_equality(adapter[10000].item(), tuple([(10000*5) + x for x in range(5)]))
        self.assert_equality(adapter[100000].item(), tuple([(100000*5) + x for x in range(5)]))
        self.assert_equality(adapter[num_records - 1].item(), tuple([((num_records - 1)*5) + x for x in range(5)]))
        #self.assert_equality(adapter[-1].item(), tuple(expected_values))

        # test 'trouble' records that have caused crashes in the past
        self.assert_equality(adapter[290000].item(), tuple([(290000*5) + x for x in range(5)]))
        self.assert_equality(adapter[818000].item(), tuple([(818000*5) + x for x in range(5)]))

        # test loading disk index
        adapter2 = iopro.CSVTextAdapter(dataz, compression='gzip', delimiter=',', field_names=False, infer_types=False, indexing=True, index_filename='test.idx')
        adapter2.set_field_types({0:'u4',1:'u4',2:'u4',3:'u4',4:'u4'})

        self.assert_equality(adapter2[0].item(), tuple([(0*5) + x for x in range(5)]))
        self.assert_equality(adapter2[10].item(), tuple([(10*5) + x for x in range(5)]))
        self.assert_equality(adapter2[100].item(), tuple([(100*5) + x for x in range(5)]))
        self.assert_equality(adapter2[1000].item(), tuple([(1000*5) + x for x in range(5)]))
        self.assert_equality(adapter2[10000].item(), tuple([(10000*5) + x for x in range(5)]))
        self.assert_equality(adapter2[100000].item(), tuple([(100000*5) + x for x in range(5)]))
        self.assert_equality(adapter2[num_records - 1].item(), tuple([((num_records - 1)*5) + x for x in range(5)]))
        #self.assert_equality(adapter[-1].item(), tuple(expected_values))

        # test 'trouble' records that have caused crashes in the past
        self.assert_equality(adapter2[290000].item(), tuple([(290000*5) + x for x in range(5)]))
        self.assert_equality(adapter2[818000].item(), tuple([(818000*5) + x for x in range(5)]))

        os.remove('test.idx')'''


    def test_header_footer(self):

        data = StringIO.StringIO('0,1,2,3,4\n5,6,7,8,9\n10,11,12,13,14')
        adapter = iopro.text_adapter(data, header=1, field_names=False)
        adapter.field_types = dict(zip(range(5), ['u4']*5))
        assert_array_equal(adapter[:], numpy.array([(5,6,7,8,9), (10,11,12,13,14)],
            dtype='u4,u4,u4,u4,u4'))

        adapter = iopro.text_adapter(data, header=2, field_names=False)
        adapter.field_types = dict(zip(range(5), ['u4']*5))
        assert_array_equal(adapter[:], numpy.array([(10,11,12,13,14)],
            dtype='u4,u4,u4,u4,u4'))

        adapter = iopro.text_adapter(data, header=1, field_names=True)
        adapter.field_types = dict(zip(range(5), ['u4']*5))
        assert_array_equal(adapter[:], numpy.array([(10,11,12,13,14)],
            dtype=[('5','u4'),('6','u4'),('7','u4'),('8','u4'),('9','u4')]))


    def test_delimiter(self):

        data = StringIO.StringIO('1,2,3\n')
        adapter = iopro.CSVTextAdapter(data, field_names=False)
        self.assert_equality(adapter[0].item(), (1,2,3))

        data = StringIO.StringIO('1 2 3\n')
        adapter = iopro.CSVTextAdapter(data, field_names=False)
        self.assert_equality(adapter[0].item(), (1,2,3))

        data = StringIO.StringIO('1\t2\t3\n')
        adapter = iopro.CSVTextAdapter(data, field_names=False)
        self.assert_equality(adapter[0].item(), (1,2,3))

        data = StringIO.StringIO('1x2x3\n')
        adapter = iopro.CSVTextAdapter(data, field_names=False)
        self.assert_equality(adapter[0].item(), (1,2,3))

        # Test no delimiter in single field csv data
        data = StringIO.StringIO('aaa\nbbb\nccc')
        array = iopro.text_adapter(data, field_names=False, delimiter=None)[:]
        assert_array_equal(array, numpy.array([('aaa',), ('bbb',), ('ccc',)], dtype=[('f0', 'O')]))


    def test_auto_type_inference(self):

        data = StringIO.StringIO('0,1,2,3,4\n5.5,6,7,8,9\n10,11,12,13,14a\n15,16,xxx,18,19')
        adapter = iopro.CSVTextAdapter(data, field_names=False, infer_types=True)
        array = adapter.to_array()
        self.assert_equality(array.dtype.fields['f0'][0], numpy.dtype('float64'))
        self.assert_equality(array.dtype.fields['f1'][0], numpy.dtype('uint64'))
        self.assert_equality(array.dtype.fields['f2'][0], numpy.dtype('O'))
        self.assert_equality(array.dtype.fields['f3'][0], numpy.dtype('uint64'))
        self.assert_equality(array.dtype.fields['f4'][0], numpy.dtype('O'))

        data = StringIO.StringIO('0,1,2,3,4\n5.5,6,7,8,9\n10,11,12,13,14a\n15,16,xxx,18,19')
        adapter = iopro.CSVTextAdapter(data, field_names=False, infer_types=True)
        self.assert_equality(adapter[0].dtype.fields['f0'][0], numpy.dtype('uint64'))
        self.assert_equality(adapter[1:3].dtype.fields['f0'][0], numpy.dtype('float64'))
        self.assert_equality(adapter[3].dtype.fields['f4'][0], numpy.dtype('uint64'))
        self.assert_equality(adapter[:].dtype.fields['f3'][0], numpy.dtype('uint64'))
        self.assert_equality(adapter[-1].dtype.fields['f2'][0], numpy.dtype('O'))
        self.assert_equality(adapter[2].dtype.fields['f4'][0], numpy.dtype('O'))

    def test_64bit_ints(self):
        
        data = StringIO.StringIO(str((2**63)-1) + ',' + str(((2**63)-1)*-1) + ',' + str((2**64)-1))
        adapter = iopro.CSVTextAdapter(data, delimiter=',', field_names=False, infer_types=False)
        adapter.set_field_types({0:'i8', 1:'i8', 2:'u8'})
        array = adapter.to_array()
        self.assert_equality(array[0].item(), ((2**63)-1, ((2**63)-1)*-1, (2**64)-1))
 
    def test_adapter_factory(self):
        data = StringIO.StringIO("1,2,3")
        adapter = iopro.text_adapter(data, "csv", delimiter=',', field_names=False, infer_types=False)
        self.assert_(isinstance(adapter, iopro.CSVTextAdapter))

        self.assertRaises(iopro.AdapterException, iopro.text_adapter, data, "foobar")

    def test_field_names(self):

        # Test for ignoring of extra fields
        data = StringIO.StringIO('f0,f1\n0,1,2\n3,4,5')
        adapter = iopro.text_adapter(data, 'csv', delimiter=',', field_names=True)
        array = adapter.to_array()
        self.assert_equality(array.dtype.names, ('f0', 'f1'))
        self.assert_equality(array[0].item(), (0,1))
        self.assert_equality(array[1].item(), (3,4))

        # Test for duplicate field names
        data = StringIO.StringIO('f0,field,field\n0,1,2\n3,4,5')
        adapter = iopro.text_adapter(data, 'csv', delimiter=',', field_names=True, infer_types=False)
        adapter.set_field_types({0:'u4', 1:'u4', 2:'u4'})
        array = adapter.to_array()
        self.assert_equality(array.dtype.names, ('f0', 'field', 'field1'))

        # Test for field names list
        data = StringIO.StringIO('0,1,2\n3,4,5')
        adapter = iopro.text_adapter(data, field_names=['a', 'b', 'c'], infer_types=False)
        adapter.field_types = {0:'u4', 1:'u4', 2:'u4'}
        array = adapter[:]
        self.assertTrue(array.dtype.names == ('a', 'b', 'c'))
        assert_array_equal(array, numpy.array([(0,1,2), (3,4,5)], dtype=[('a', 'u4'), ('b', 'u4'), ('c', 'u4')]))

    def test_float_conversion(self):
        data = StringIO.StringIO('10,1.333,-1.23,10.0E+2,999.9e-2')
        adapter = iopro.text_adapter(data, field_names=False, infer_types=False)
        adapter.set_field_types(dict(zip(range(5), ['f8']*5)))
        array = adapter[0]
        #self.assert_equality(array[0].item(), (10.0,1.333,-1.23,1000.0,9.999))
        self.assertAlmostEqual(array[0][0], 10.0)
        self.assertAlmostEqual(array[0][1], 1.333)
        self.assertAlmostEqual(array[0][2], -1.23)
        self.assertAlmostEqual(array[0][3], 1000.0)
        self.assertAlmostEqual(array[0][4], 9.999)

    def test_generators(self):

        def int_generator(num_recs):
            for i in range(num_recs):
                yield ','.join([str(i*5), str(i*5+1), str(i*5+2), str(i*5+3), str(i*5+4)])

        adapter = iopro.text_adapter(int_generator(self.num_records), field_names=False)
        array = adapter[:]

        self.assert_equality(array.size, self.num_records)
        
        record = [x for x in range(0, 5)]
        for i in range(0, self.num_records):
            self.assert_equality(array[i].item(), tuple(record))
            record[0] += 5
            record[1] += 5
            record[2] += 5
            record[3] += 5
            record[4] += 5

    def test_comments(self):

        data = StringIO.StringIO('1,2,3\n#4,5,6')
        adapter = iopro.text_adapter(data, field_names=False)
        array = adapter[:]
        self.assert_equality(array.size, 1)
        self.assert_equality(array[0].item(), (1,2,3))

        data = StringIO.StringIO('1,2,3\n#4,5,6')
        adapter = iopro.text_adapter(data, field_names=False, comment=None)
        array = adapter[:]
        self.assert_equality(array.size, 2)
        self.assert_equality(array[0].item(), ('1',2,3))
        self.assert_equality(array[1].item(), ('#4',5,6))

    def test_escapechar(self):

        data = StringIO.StringIO('1,2\\2,3\n4,5\\5\\5,6')
        array = iopro.text_adapter(data, field_names=False)[:]
        assert_array_equal(array,
            numpy.array([(1,22,3), (4,555,6)], dtype='u8,u8,u8'))

        data = StringIO.StringIO('\\1,2,3\n4,5,6\\')
        array = iopro.text_adapter(data, field_names=False)[:]
        assert_array_equal(array,
            numpy.array([(1,2,3), (4,5,6)], dtype='u8,u8,u8'))

        data = StringIO.StringIO('a,b\\,b,c\na,b\\,b\\,b,c')
        array = iopro.text_adapter(data, field_names=False)[:]
        assert_array_equal(array,
            numpy.array([('a', 'b,b', 'c'), ('a', 'b,b,b', 'c')], dtype='O,O,O'))

        data = StringIO.StringIO('a,bx,b,c\na,bx,bx,b,c')
        array = iopro.text_adapter(data, field_names=False, escape='x')[:]
        assert_array_equal(array,
            numpy.array([('a', 'b,b', 'c'), ('a', 'b,b,b', 'c')], dtype='O,O,O'))

    '''def test_dataframe_output(self):

        try:
            import pandas
        except ImportError:
            return

        # Test filling blank lines with fill values if output is dataframe
        data = StringIO.StringIO('1,2,3\n\n4,5,6')
        adapter = iopro.text_adapter(data, field_names=False)
        adapter.field_types = {0:'O', 1:'O', 2:'O'}
        adapter.set_fill_values({0:numpy.nan, 1:numpy.nan, 2:numpy.nan})
        df = adapter.to_dataframe()'''

    def test_csv(self):

        # Test skipping blank lines
        data = StringIO.StringIO('1,2,3\n\n4,5,6')
        adapter = iopro.text_adapter(data, field_names=False)
        array = adapter[:]
        assert_array_equal(array, numpy.array([(1,2,3), (4,5,6)],
            dtype=[('f0','<u8'),('f1','<u8'),('f2','<u8')]))

    def test_json(self):

        # Test json number
        data = StringIO.StringIO('{"id":123}')
        adapter = iopro.text_adapter(data, parser='json')
        array = adapter[:]
        assert_array_equal(array, numpy.array([(123,)], dtype=[('f0', 'u8')]))

        # Test json number
        data = StringIO.StringIO('{"id":"xxx"}')
        adapter = iopro.text_adapter(data, parser='json')
        array = adapter[:]
        assert_array_equal(array, numpy.array([('xxx',)], dtype=[('f0', 'O')]))

        # Test multiple values
        data = StringIO.StringIO('{"id":123, "name":"xxx"}')
        adapter = iopro.text_adapter(data, parser='json')
        array = adapter[:]
        assert_array_equal(array, numpy.array([(123, 'xxx',)], dtype=[('f0', 'u8'), ('f1', 'O')]))

        # Test multiple records
        data = StringIO.StringIO('[{"id":123, "name":"xxx"}, {"id":456, "name":"yyy"}]')
        adapter = iopro.text_adapter(data, parser='json')
        array = adapter[:]
        assert_array_equal(array, numpy.array([(123, 'xxx',), (456, 'yyy')], dtype=[('f0', 'u8'), ('f1', 'O')]))

        # Test multiple objects separated by newlines
        data = StringIO.StringIO('{"id":123, "name":"xxx"}\n{"id":456, "name":"yyy"}')
        adapter = iopro.text_adapter(data, parser='json')
        array = adapter[:]
        assert_array_equal(array, numpy.array([(123, 'xxx',), (456, 'yyy')], dtype=[('f0', 'u8'), ('f1', 'O')]))

        data = StringIO.StringIO('{"id":123, "name":"xxx"}\n')
        adapter = iopro.text_adapter(data, parser='json')
        array = adapter[:]
        assert_array_equal(array, numpy.array([(123, 'xxx',)], dtype=[('f0', 'u8'), ('f1', 'O')]))

        # Test subarrays
        data = StringIO.StringIO('{"id":123, "names":["xxx","yyy","zzz"]}')
        adapter = iopro.text_adapter(data, parser='json')
        array = adapter[:]
        assert_array_equal(array, numpy.array([(123, 'xxx', 'yyy', 'zzz',)],
            dtype=[('f0', 'u8'), ('f1', 'O'), ('f2', 'O'), ('f3', 'O')]))

        # Test subobjects
        data = StringIO.StringIO('{"id":123, "names":{"a":"xxx", "b":"yyy", "c":"zzz"}}')
        adapter = iopro.text_adapter(data, parser='json')
        array = adapter[:]
        assert_array_equal(array, numpy.array([(123, 'xxx', 'yyy', 'zzz',)],
            dtype=[('f0', 'u8'), ('f1', 'O'), ('f2', 'O'), ('f3', 'O')]))

def run(verbosity=1, num_records=100000):
    if num_records < 10:
        raise ValueError('number of records for generated datasets must be at least 10')
    TestTextAdapter.num_records = num_records
    suite = unittest.TestLoader().loadTestsFromTestCase(TestTextAdapter)
    return unittest.TextTestRunner(verbosity=verbosity).run(suite)


if __name__ == '__main__':
    run()
