From afdd89f93dbc9df9a13f8d48685f3ea7a29d1da0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mikkel=20Elle=20Lepper=C3=B8d?= Date: Fri, 10 May 2019 17:32:58 +0200 Subject: [PATCH 1/8] Implement soft link and external link --- exdir/__init__.py | 5 +- exdir/core/__init__.py | 1 + exdir/core/constants.py | 9 ++ exdir/core/exdir_file.py | 6 ++ exdir/core/exdir_object.py | 26 +++++- exdir/core/group.py | 22 ++++- exdir/core/links.py | 58 +++++++++++++ tests/test_links.py | 170 +++++++++++++++++++++++++++++++++++++ 8 files changed, 294 insertions(+), 3 deletions(-) create mode 100644 exdir/core/links.py create mode 100644 tests/test_links.py diff --git a/exdir/__init__.py b/exdir/__init__.py index 2092ccb..16e7125 100644 --- a/exdir/__init__.py +++ b/exdir/__init__.py @@ -1,7 +1,10 @@ from . import core from . import plugin_interface from . import plugins -from .core import File, validation, Attribute, Dataset, Group, Raw, Object +from .core import ( + File, validation, Attribute, Dataset, Group, Raw, Object, SoftLink, + ExternalLink +) # TODO remove versioneer from ._version import get_versions diff --git a/exdir/core/__init__.py b/exdir/core/__init__.py index 474ff48..11caae6 100644 --- a/exdir/core/__init__.py +++ b/exdir/core/__init__.py @@ -11,3 +11,4 @@ from .dataset import Dataset from .group import Group from .raw import Raw +from .links import SoftLink, ExternalLink diff --git a/exdir/core/constants.py b/exdir/core/constants.py index eb833aa..8edda4f 100644 --- a/exdir/core/constants.py +++ b/exdir/core/constants.py @@ -2,6 +2,15 @@ EXDIR_METANAME = "exdir" TYPE_METANAME = "type" VERSION_METANAME = "version" +LINK_METANAME = "link" +TARGET_METANAME = "target" + +#links +LINK_TYPENAME = "link" +LINK_TARGETNAME = "target" +LINK_EXTERNALNAME = "external" +LINK_SOFTNAME = "soft" +LINK_FILENAME = "file" # filenames META_FILENAME = "exdir.yaml" diff --git a/exdir/core/exdir_file.py b/exdir/core/exdir_file.py index becb2a9..06b45c2 100644 --- a/exdir/core/exdir_file.py +++ b/exdir/core/exdir_file.py @@ -179,6 +179,12 @@ def __getitem__(self, name): return self return super(File, self).__getitem__(path) + def __setitem__(self, name, value): + path = utils.path.remove_root(name) + if len(path.parts) < 1: + return self + return super(File, self).__setitem__(path, value) + def __contains__(self, name): path = utils.path.remove_root(name) return super(File, self).__contains__(path) diff --git a/exdir/core/exdir_object.py b/exdir/core/exdir_object.py index 8592195..e65462f 100644 --- a/exdir/core/exdir_object.py +++ b/exdir/core/exdir_object.py @@ -113,7 +113,8 @@ def is_nonraw_object_directory(directory): return False if TYPE_METANAME not in meta_data[EXDIR_METANAME]: return False - valid_types = [DATASET_TYPENAME, FILE_TYPENAME, GROUP_TYPENAME] + valid_types = [ + DATASET_TYPENAME, FILE_TYPENAME, GROUP_TYPENAME, LINK_METANAME] if meta_data[EXDIR_METANAME][TYPE_METANAME] not in valid_types: return False return True @@ -123,6 +124,29 @@ def is_raw_object_directory(directory): return is_exdir_object(directory) and not is_nonraw_object_directory(directory) +def is_link_object_directory(directory): + if not is_exdir_object(directory): + return False + meta_filename = directory / META_FILENAME + if not meta_filename.exists(): + return False + with meta_filename.open("r", encoding="utf-8") as meta_file: + meta_data = yaml.safe_load(meta_file) + + if not isinstance(meta_data, dict): + return False + + if EXDIR_METANAME not in meta_data: + return False + + if TYPE_METANAME not in meta_data[EXDIR_METANAME]: + return False + + if meta_data[EXDIR_METANAME][TYPE_METANAME] == LINK_METANAME: + return True + return False + + def root_directory(path): """ Iterates upwards until a exdir.File object is found. diff --git a/exdir/core/group.py b/exdir/core/group.py index 8eaf3e4..5d676cb 100644 --- a/exdir/core/group.py +++ b/exdir/core/group.py @@ -19,11 +19,14 @@ import collections as abc from .exdir_object import Object +from .links import Link from . import exdir_object as exob +from . import exdir_file as exfile from . import dataset as ds from . import raw from .. import utils + def _data_to_shape_and_dtype(data, shape, dtype): if data is not None: if shape is None: @@ -35,6 +38,7 @@ def _data_to_shape_and_dtype(data, shape, dtype): dtype = np.float32 return shape, dtype + def _assert_data_shape_dtype_match(data, shape, dtype): if data is not None: if shape is not None and np.product(shape) != np.product(data.shape): @@ -52,6 +56,7 @@ def _assert_data_shape_dtype_match(data, shape, dtype): ) return + class Group(Object): """ Container of other groups and datasets. @@ -387,6 +392,14 @@ def __getitem__(self, name): directory = self.directory / path + if exob.is_link_object_directory(directory): + link_meta = self[name].meta[LINK_METANAME] + if link_meta[TYPE_METANAME] == LINK_SOFTNAME: + result = self[link_meta[LINK_TARGETNAME]] + elif link_meta[TYPE_METANAME] == LINK_EXTERNALNAME: + result = exfile.File(link_meta[LINK_FILENAME], 'r')[LINK_TARGETNAME] + return result + if exob.is_raw_object_directory(directory): # TODO create one function that handles all Raw creation return raw.Raw( root_directory=self.root_directory, @@ -446,6 +459,13 @@ def __setitem__(self, name, value): self[path.parent][path.name] = value return + if isinstance(value, Link): + link_group = self.create_group(name) + if value.path not in self.file: + return # TODO works when merging with lepmik/close + link_group.meta.update(value._link) + return + if name not in self: self.create_dataset(name, data=value) return @@ -513,7 +533,7 @@ def __len__(self): return len([a for a in self]) - def get(self, key): + def get(self, key, getLink=False): """ Get an object in the group. Parameters diff --git a/exdir/core/links.py b/exdir/core/links.py new file mode 100644 index 0000000..b4cd55a --- /dev/null +++ b/exdir/core/links.py @@ -0,0 +1,58 @@ +from . import exdir_file +from .exdir_object import Object +from .constants import * + + +class Link(Object): + """ + Super class for link objects + """ + def __init__(self, path): + self.path = path + + @property + def _link(self): + return {TYPE_METANAME: LINK_TYPENAME} + + def __eq__(self, other): + assert self._link.get(LINK_METANAME) == other._link.get(LINK_METANAME) + + +class SoftLink(Link): + def __init__(self, path): + super(SoftLink, self).__init__( + path=path + ) + + @property + def _link(self): + result = { + TYPE_METANAME: LINK_TYPENAME, + LINK_METANAME: { + TYPE_METANAME: LINK_SOFTNAME, + LINK_TARGETNAME: self.path + } + } + return result + + +class ExternalLink(Link): + def __init__(self, other_exdir_path, path): + super(ExternalLink, self).__init__( + path=path + ) + self.other_exdir_path = other_exdir_path + # with exdir_file.File(self.other_exdir_path) as f: + # pass + + @property + def _link(self): + result = { + TYPE_METANAME: LINK_TYPENAME, + LINK_METANAME: { + TYPE_METANAME: LINK_EXTERNALNAME, + LINK_TARGETNAME: self.path, + LINK_FILENAME: self.other_exdir_path + } + } + return result diff --git a/tests/test_links.py b/tests/test_links.py new file mode 100644 index 0000000..c59036b --- /dev/null +++ b/tests/test_links.py @@ -0,0 +1,170 @@ +# -*- coding: utf-8 -*- + +# This file is part of Exdir, the Experimental Directory Structure. +# +# Copyright 2019 Mikkel Lepperød +# +# License: MIT, see "LICENSE" file for the full license terms. +# +# This file contains code from h5py, a Python interface to the HDF5 library, +# licensed under a standard 3-clause BSD license +# with copyright Andrew Collette and contributors. +# See http://www.h5py.org and the "3rdparty/h5py-LICENSE" file for details. + +import exdir +import pytest +import numpy as np +try: + import ruamel_yaml as yaml +except ImportError: + import ruamel.yaml as yaml + + +def test_softlinks(setup_teardown_file): + """ Broken softlinks are contained, but their members are not """ + f = setup_teardown_file[3] + g = exdir.File(setup_teardown_file[2] / 'mongoose.exdir') + f.create_group('mongoose') + g.create_group('mongoose') + f.create_group('grp') + f['/grp/soft'] = exdir.SoftLink('/mongoose') + f['/grp/external'] = exdir.ExternalLink('mongoose.exdir', '/mongoose') + assert '/grp/soft' in f + assert '/grp/soft/something' not in f + assert '/grp/external' in f + assert '/grp/external/something' not in f + + +# def test_get_link(setup_teardown_file): +# """ Get link values """ +# f = setup_teardown_file[3] +# g = exdir.File(setup_teardown_file[0] / 'mongoose.exdir') +# f.create_group('mongoose') +# g.create_group('mongoose') +# sl = SoftLink('/mongoose') +# el = ExternalLink('somewhere.hdf5', 'mongoose') +# +# f['soft'] = sl +# f['external'] = el +# +# out_sl = f.get('soft', getlink=True) +# out_el = f.get('external', getlink=True) +# +# #TODO: redo with SoftLink/ExternalLink built-in equality +# assertIsInstance(out_sl, SoftLink) +# assert out_sl == sl +# assertIsInstance(out_el, ExternalLink) +# assert out_el == el +# +# +# Feature: Create and manage soft links with the high-level interface +def test_spath(setup_teardown_file): + """ SoftLink directory attribute """ + sl = exdir.SoftLink('/foo') + assert sl.path == '/foo' + + +# def test_srepr(setup_teardown_file): +# """ SoftLink path repr """ +# sl = SoftLink('/foo') +# assertIsInstance(repr(sl), six.string_types) + + +def test_create(setup_teardown_file): + """ Create new soft link by assignment """ + f = setup_teardown_file[3] + g = f.create_group('new') + sl = exdir.SoftLink('/new') + f['alias'] = sl + g2 = f['alias'] + assert g == g2 + + +def test_exc(setup_teardown_file): + """ Opening dangling soft link results in KeyError """ + f = setup_teardown_file[3] + f['alias'] = exdir.SoftLink('new') + with pytest.raises(KeyError): + f['alias'] +# +# +# # Feature: Create and manage external links +# def test_epath(setup_teardown_file): +# """ External link paths attributes """ +# el = ExternalLink('foo.hdf5', '/foo') +# assertEqual(el.filename, 'foo.hdf5') +# assertEqual(el.path, '/foo') +# +# def test_erepr(setup_teardown_file): +# """ External link repr """ +# el = ExternalLink('foo.hdf5','/foo') +# assertIsInstance(repr(el), six.string_types) +# +# def test_create(setup_teardown_file): +# """ Creating external links """ +# f['ext'] = ExternalLink(ename, '/external') +# grp = f['ext'] +# ef = grp.file +# assertNotEqual(ef, f) +# assertEqual(grp.name, '/external') +# +# def test_exc(setup_teardown_file): +# """ KeyError raised when attempting to open broken link """ +# f['ext'] = ExternalLink(ename, '/missing') +# with assertRaises(KeyError): +# f['ext'] +# +# # I would prefer IOError but there's no way to fix this as the exception +# # class is determined by HDF5. +# def test_exc_missingfile(setup_teardown_file): +# """ KeyError raised when attempting to open missing file """ +# f['ext'] = ExternalLink('mongoose.hdf5','/foo') +# with assertRaises(KeyError): +# f['ext'] +# +# def test_close_file(setup_teardown_file): +# """ Files opened by accessing external links can be closed +# Issue 189. +# """ +# f['ext'] = ExternalLink(ename, '/') +# grp = f['ext'] +# f2 = grp.file +# f2.close() +# assertFalse(f2) +# +# +# def test_unicode_encode(setup_teardown_file): +# """ +# Check that external links encode unicode filenames properly +# Testing issue #732 +# """ +# ext_filename = os.path.join(mkdtemp(), u"α.hdf5") +# with File(ext_filename, "w") as ext_file: +# ext_file.create_group('external') +# f['ext'] = ExternalLink(ext_filename, '/external') +# +# +# def test_unicode_decode(setup_teardown_file): +# """ +# Check that external links decode unicode filenames properly +# Testing issue #732 +# """ +# ext_filename = os.path.join(mkdtemp(), u"α.hdf5") +# with File(ext_filename, "w") as ext_file: +# ext_file.create_group('external') +# ext_file["external"].attrs["ext_attr"] = "test" +# f['ext'] = ExternalLink(ext_filename, '/external') +# assertEqual(f["ext"].attrs["ext_attr"], "test") +# +# +# def test_unicode_hdf5_path(setup_teardown_file): +# """ +# Check that external links handle unicode hdf5 paths properly +# Testing issue #333 +# """ +# ext_filename = os.path.join(mkdtemp(), "external.hdf5") +# with File(ext_filename, "w") as ext_file: +# ext_file.create_group(u'α') +# ext_file[u"α"].attrs["ext_attr"] = "test" +# f['ext'] = ExternalLink(ext_filename, u'/α') +# assertEqual(f["ext"].attrs["ext_attr"], "test") From a5742af948d271a1a9c810dda2ce5769b87d799d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mikkel=20E=20Lepper=C3=B8d?= Date: Sat, 11 May 2019 13:11:26 +0200 Subject: [PATCH 2/8] Check if is link directly in group.__getitem__ and more tests --- exdir/core/exdir_object.py | 23 ------------- exdir/core/group.py | 48 ++++++++++++++++---------- exdir/core/links.py | 8 ++++- tests/test_links.py | 69 ++++++++++++++++++++------------------ 4 files changed, 75 insertions(+), 73 deletions(-) diff --git a/exdir/core/exdir_object.py b/exdir/core/exdir_object.py index e65462f..11913ea 100644 --- a/exdir/core/exdir_object.py +++ b/exdir/core/exdir_object.py @@ -124,29 +124,6 @@ def is_raw_object_directory(directory): return is_exdir_object(directory) and not is_nonraw_object_directory(directory) -def is_link_object_directory(directory): - if not is_exdir_object(directory): - return False - meta_filename = directory / META_FILENAME - if not meta_filename.exists(): - return False - with meta_filename.open("r", encoding="utf-8") as meta_file: - meta_data = yaml.safe_load(meta_file) - - if not isinstance(meta_data, dict): - return False - - if EXDIR_METANAME not in meta_data: - return False - - if TYPE_METANAME not in meta_data[EXDIR_METANAME]: - return False - - if meta_data[EXDIR_METANAME][TYPE_METANAME] == LINK_METANAME: - return True - return False - - def root_directory(path): """ Iterates upwards until a exdir.File object is found. diff --git a/exdir/core/group.py b/exdir/core/group.py index 5d676cb..d7d4c85 100644 --- a/exdir/core/group.py +++ b/exdir/core/group.py @@ -19,7 +19,7 @@ import collections as abc from .exdir_object import Object -from .links import Link +from .links import Link, SoftLink, ExternalLink from . import exdir_object as exob from . import exdir_file as exfile from . import dataset as ds @@ -392,14 +392,6 @@ def __getitem__(self, name): directory = self.directory / path - if exob.is_link_object_directory(directory): - link_meta = self[name].meta[LINK_METANAME] - if link_meta[TYPE_METANAME] == LINK_SOFTNAME: - result = self[link_meta[LINK_TARGETNAME]] - elif link_meta[TYPE_METANAME] == LINK_EXTERNALNAME: - result = exfile.File(link_meta[LINK_FILENAME], 'r')[LINK_TARGETNAME] - return result - if exob.is_raw_object_directory(directory): # TODO create one function that handles all Raw creation return raw.Raw( root_directory=self.root_directory, @@ -422,6 +414,8 @@ def __getitem__(self, name): return self._dataset(name) elif meta_data[exob.EXDIR_METANAME][exob.TYPE_METANAME] == exob.GROUP_TYPENAME: return self._group(name) + elif meta_data[exob.EXDIR_METANAME][exob.TYPE_METANAME] == exob.LINK_TYPENAME: + return self._link(name) else: error_string = ( "Object {name} has data type {type}.\n" @@ -432,6 +426,24 @@ def __getitem__(self, name): ) raise NotImplementedError(error_string) + def _link(self, name, get_link=False): + link_meta = self._group(name).meta[exob.EXDIR_METANAME][exob.LINK_METANAME] + print(link_meta) + if link_meta[exob.TYPE_METANAME] == exob.LINK_SOFTNAME: + if get_link: + result = SoftLink(link_meta[exob.LINK_TARGETNAME]) + else: + result = self[link_meta[exob.LINK_TARGETNAME]] + elif link_meta[exob.TYPE_METANAME] == exob.LINK_EXTERNALNAME: + if get_link: + result = ExternalLink( + link_meta[exob.LINK_FILENAME], + link_meta[exob.LINK_TARGETNAME]) + else: + result = exfile.File( + link_meta[exob.LINK_FILENAME], 'r')[exob.LINK_TARGETNAME] + return result + def _dataset(self, name): return ds.Dataset( root_directory=self.root_directory, @@ -461,9 +473,9 @@ def __setitem__(self, name, value): if isinstance(value, Link): link_group = self.create_group(name) - if value.path not in self.file: - return # TODO works when merging with lepmik/close - link_group.meta.update(value._link) + # if value.path not in self.file: + # return # TODO works when merging with lepmik/close + link_group.meta[exob.EXDIR_METANAME].update(value._link) return if name not in self: @@ -533,19 +545,21 @@ def __len__(self): return len([a for a in self]) - def get(self, key, getLink=False): + def get(self, name, get_link=False): """ Get an object in the group. Parameters ---------- - key : str - The key of the desired object + name : str + The name of the desired object Returns ------- Value or None if object does not exist. """ - if key in self: - return self[key] + if name in self: + if get_link: + return self._link(name, get_link) + return self[name] else: return None diff --git a/exdir/core/links.py b/exdir/core/links.py index b4cd55a..9c659d1 100644 --- a/exdir/core/links.py +++ b/exdir/core/links.py @@ -15,7 +15,7 @@ def _link(self): return {TYPE_METANAME: LINK_TYPENAME} def __eq__(self, other): - assert self._link.get(LINK_METANAME) == other._link.get(LINK_METANAME) + return self._link.get(LINK_METANAME) == other._link.get(LINK_METANAME) class SoftLink(Link): @@ -35,6 +35,9 @@ def _link(self): } return result + def __repr__(self): + return "Exdir SoftLink '{}' at {}".format(self.path, id(self)) + class ExternalLink(Link): def __init__(self, other_exdir_path, path): @@ -56,3 +59,6 @@ def _link(self): } } return result + + def __repr__(self): + return "Exdir SoftLink '{}' at {}".format(self.path, id(self)) diff --git a/tests/test_links.py b/tests/test_links.py index c59036b..7b23b6d 100644 --- a/tests/test_links.py +++ b/tests/test_links.py @@ -11,7 +11,7 @@ # with copyright Andrew Collette and contributors. # See http://www.h5py.org and the "3rdparty/h5py-LICENSE" file for details. -import exdir +from exdir import SoftLink, ExternalLink, File import pytest import numpy as np try: @@ -20,47 +20,52 @@ import ruamel.yaml as yaml -def test_softlinks(setup_teardown_file): +def test_soft_links(setup_teardown_file): """ Broken softlinks are contained, but their members are not """ f = setup_teardown_file[3] - g = exdir.File(setup_teardown_file[2] / 'mongoose.exdir') f.create_group('mongoose') - g.create_group('mongoose') f.create_group('grp') - f['/grp/soft'] = exdir.SoftLink('/mongoose') - f['/grp/external'] = exdir.ExternalLink('mongoose.exdir', '/mongoose') + f['/grp/soft'] = SoftLink('/mongoose') assert '/grp/soft' in f assert '/grp/soft/something' not in f + + +def test_external_links(setup_teardown_file): + """ Broken softlinks are contained, but their members are not """ + f = setup_teardown_file[3] + g = File(setup_teardown_file[0] / 'mongoose.exdir', 'w') + g.create_group('mongoose') + f.create_group('grp') + f['/grp/external'] = ExternalLink('mongoose.exdir', '/mongoose') assert '/grp/external' in f assert '/grp/external/something' not in f -# def test_get_link(setup_teardown_file): -# """ Get link values """ -# f = setup_teardown_file[3] -# g = exdir.File(setup_teardown_file[0] / 'mongoose.exdir') -# f.create_group('mongoose') -# g.create_group('mongoose') -# sl = SoftLink('/mongoose') -# el = ExternalLink('somewhere.hdf5', 'mongoose') -# -# f['soft'] = sl -# f['external'] = el -# -# out_sl = f.get('soft', getlink=True) -# out_el = f.get('external', getlink=True) -# -# #TODO: redo with SoftLink/ExternalLink built-in equality -# assertIsInstance(out_sl, SoftLink) -# assert out_sl == sl -# assertIsInstance(out_el, ExternalLink) -# assert out_el == el -# -# +def test_get_link(setup_teardown_file): + """ Get link values """ + f = setup_teardown_file[3] + g = File(setup_teardown_file[0] / 'mongoose.exdir') + f.create_group('mongoose') + g.create_group('mongoose') + sl = SoftLink('/mongoose') + el = ExternalLink('somewhere.hdf5', 'mongoose') + + f['soft'] = sl + f['external'] = el + + out_sl = f.get('soft', get_link=True) + out_el = f.get('external', get_link=True) + + assert isinstance(out_sl, SoftLink) + assert out_sl == sl + assert isinstance(out_el, ExternalLink) + assert out_el == el + + # Feature: Create and manage soft links with the high-level interface def test_spath(setup_teardown_file): """ SoftLink directory attribute """ - sl = exdir.SoftLink('/foo') + sl = SoftLink('/foo') assert sl.path == '/foo' @@ -70,11 +75,11 @@ def test_spath(setup_teardown_file): # assertIsInstance(repr(sl), six.string_types) -def test_create(setup_teardown_file): +def test_linked_group_equal(setup_teardown_file): """ Create new soft link by assignment """ f = setup_teardown_file[3] g = f.create_group('new') - sl = exdir.SoftLink('/new') + sl = SoftLink('/new') f['alias'] = sl g2 = f['alias'] assert g == g2 @@ -83,7 +88,7 @@ def test_create(setup_teardown_file): def test_exc(setup_teardown_file): """ Opening dangling soft link results in KeyError """ f = setup_teardown_file[3] - f['alias'] = exdir.SoftLink('new') + f['alias'] = SoftLink('new') with pytest.raises(KeyError): f['alias'] # From 7eb782d6b2f378e9ac4da5cb96729098ba399b07 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mikkel=20E=20Lepper=C3=B8d?= Date: Sat, 11 May 2019 14:17:24 +0200 Subject: [PATCH 3/8] Testing of externallink, need to merge with dev to get acces to file --- exdir/core/group.py | 5 ++- exdir/core/links.py | 10 ++--- tests/test_links.py | 97 ++++++++++++++++++++++++++------------------- 3 files changed, 65 insertions(+), 47 deletions(-) diff --git a/exdir/core/group.py b/exdir/core/group.py index d7d4c85..00a98c1 100644 --- a/exdir/core/group.py +++ b/exdir/core/group.py @@ -440,8 +440,9 @@ def _link(self, name, get_link=False): link_meta[exob.LINK_FILENAME], link_meta[exob.LINK_TARGETNAME]) else: - result = exfile.File( - link_meta[exob.LINK_FILENAME], 'r')[exob.LINK_TARGETNAME] + external_file = exfile.File( + link_meta[exob.LINK_FILENAME], 'r') + result = external_file[link_meta[exob.LINK_TARGETNAME]] return result def _dataset(self, name): diff --git a/exdir/core/links.py b/exdir/core/links.py index 9c659d1..931cea1 100644 --- a/exdir/core/links.py +++ b/exdir/core/links.py @@ -40,13 +40,13 @@ def __repr__(self): class ExternalLink(Link): - def __init__(self, other_exdir_path, path): + def __init__(self, filename, path): super(ExternalLink, self).__init__( path=path ) - self.other_exdir_path = other_exdir_path - # with exdir_file.File(self.other_exdir_path) as f: - # pass + self.filename = filename + with exdir_file.File(self.filename) as f: + pass @property def _link(self): @@ -55,7 +55,7 @@ def _link(self): LINK_METANAME: { TYPE_METANAME: LINK_EXTERNALNAME, LINK_TARGETNAME: self.path, - LINK_FILENAME: self.other_exdir_path + LINK_FILENAME: str(self.filename) } } return result diff --git a/tests/test_links.py b/tests/test_links.py index 7b23b6d..da66ad9 100644 --- a/tests/test_links.py +++ b/tests/test_links.py @@ -44,11 +44,11 @@ def test_external_links(setup_teardown_file): def test_get_link(setup_teardown_file): """ Get link values """ f = setup_teardown_file[3] - g = File(setup_teardown_file[0] / 'mongoose.exdir') + g = File(setup_teardown_file[0] / 'somewhere.exdir') f.create_group('mongoose') g.create_group('mongoose') sl = SoftLink('/mongoose') - el = ExternalLink('somewhere.hdf5', 'mongoose') + el = ExternalLink('somewhere.exdir', 'mongoose') f['soft'] = sl f['external'] = el @@ -63,16 +63,16 @@ def test_get_link(setup_teardown_file): # Feature: Create and manage soft links with the high-level interface -def test_spath(setup_teardown_file): +def test_soft_path(setup_teardown_file): """ SoftLink directory attribute """ sl = SoftLink('/foo') assert sl.path == '/foo' -# def test_srepr(setup_teardown_file): -# """ SoftLink path repr """ -# sl = SoftLink('/foo') -# assertIsInstance(repr(sl), six.string_types) +def test_soft_repr(setup_teardown_file): + """ SoftLink path repr """ + sl = SoftLink('/foo') + assert isinstance(repr(sl), str) def test_linked_group_equal(setup_teardown_file): @@ -91,39 +91,56 @@ def test_exc(setup_teardown_file): f['alias'] = SoftLink('new') with pytest.raises(KeyError): f['alias'] -# -# -# # Feature: Create and manage external links -# def test_epath(setup_teardown_file): -# """ External link paths attributes """ -# el = ExternalLink('foo.hdf5', '/foo') -# assertEqual(el.filename, 'foo.hdf5') -# assertEqual(el.path, '/foo') -# -# def test_erepr(setup_teardown_file): -# """ External link repr """ -# el = ExternalLink('foo.hdf5','/foo') -# assertIsInstance(repr(el), six.string_types) -# -# def test_create(setup_teardown_file): -# """ Creating external links """ -# f['ext'] = ExternalLink(ename, '/external') -# grp = f['ext'] -# ef = grp.file -# assertNotEqual(ef, f) -# assertEqual(grp.name, '/external') -# -# def test_exc(setup_teardown_file): -# """ KeyError raised when attempting to open broken link """ -# f['ext'] = ExternalLink(ename, '/missing') -# with assertRaises(KeyError): -# f['ext'] + + +# Feature: Create and manage external links +def test_external_path(setup_teardown_file): + """ External link paths attributes """ + g = File(setup_teardown_file[0] / 'foo.exdir', 'w') + egrp = g.create_group('foo') + el = ExternalLink(setup_teardown_file[0] / 'foo.exdir', '/foo') + assert el.filename == 'foo.exdir' + assert el.path == '/foo' + + +def test_external_must_exist(setup_teardown_file): + """ External link paths attributes """ + with pytest.raises(FileExistsError): + el = ExternalLink('foo.exdir', '/foo') + + +def test_external_repr(setup_teardown_file): + """ External link repr """ + g = File(setup_teardown_file[0] / 'foo.exdir', 'w') + el = ExternalLink(setup_teardown_file[0] / 'foo.exdir', '/foo') + assert isinstance(repr(el), str) + + +def test_create(setup_teardown_file): + """ Creating external links """ + f = setup_teardown_file[3] + g = File(setup_teardown_file[0] / 'foo.exdir', 'w') + egrp = g.require_group('external') + f['ext'] = ExternalLink(setup_teardown_file[0] / 'foo.exdir', '/external') + grp = f['ext'] + ef = grp.file + assert ef != f + assert grp.name == '/external' + + +def test_broken_external_link(setup_teardown_file): + """ KeyError raised when attempting to open broken link """ + f = setup_teardown_file[3] + g = File(setup_teardown_file[0] / 'foo.exdir', 'w') + f['ext'] = ExternalLink(setup_teardown_file[0] / 'foo.exdir', '/missing') + with pytest.raises(KeyError): + f['ext'] # # # I would prefer IOError but there's no way to fix this as the exception # # class is determined by HDF5. # def test_exc_missingfile(setup_teardown_file): # """ KeyError raised when attempting to open missing file """ -# f['ext'] = ExternalLink('mongoose.hdf5','/foo') +# f['ext'] = ExternalLink('mongoose.exdir','/foo') # with assertRaises(KeyError): # f['ext'] # @@ -143,7 +160,7 @@ def test_exc(setup_teardown_file): # Check that external links encode unicode filenames properly # Testing issue #732 # """ -# ext_filename = os.path.join(mkdtemp(), u"α.hdf5") +# ext_filename = os.path.join(mkdtemp(), u"α.exdir") # with File(ext_filename, "w") as ext_file: # ext_file.create_group('external') # f['ext'] = ExternalLink(ext_filename, '/external') @@ -154,7 +171,7 @@ def test_exc(setup_teardown_file): # Check that external links decode unicode filenames properly # Testing issue #732 # """ -# ext_filename = os.path.join(mkdtemp(), u"α.hdf5") +# ext_filename = os.path.join(mkdtemp(), u"α.exdir") # with File(ext_filename, "w") as ext_file: # ext_file.create_group('external') # ext_file["external"].attrs["ext_attr"] = "test" @@ -162,12 +179,12 @@ def test_exc(setup_teardown_file): # assertEqual(f["ext"].attrs["ext_attr"], "test") # # -# def test_unicode_hdf5_path(setup_teardown_file): +# def test_unicode_exdir_path(setup_teardown_file): # """ -# Check that external links handle unicode hdf5 paths properly +# Check that external links handle unicode exdir paths properly # Testing issue #333 # """ -# ext_filename = os.path.join(mkdtemp(), "external.hdf5") +# ext_filename = os.path.join(mkdtemp(), "external.exdir") # with File(ext_filename, "w") as ext_file: # ext_file.create_group(u'α') # ext_file[u"α"].attrs["ext_attr"] = "test" From d9fbbdcd49a8586eeda118e293c4102250ab26cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mikkel=20E=20Lepper=C3=B8d?= Date: Sat, 11 May 2019 14:47:35 +0200 Subject: [PATCH 4/8] Properly test external links --- exdir/core/exdir_file.py | 2 +- exdir/core/links.py | 11 +++-- tests/test_links.py | 94 ++++++++++++++++++++-------------------- 3 files changed, 55 insertions(+), 52 deletions(-) diff --git a/exdir/core/exdir_file.py b/exdir/core/exdir_file.py index 60b7f99..097e29b 100644 --- a/exdir/core/exdir_file.py +++ b/exdir/core/exdir_file.py @@ -71,7 +71,7 @@ class File(Group): def __init__(self, directory, mode=None, allow_remove=False, name_validation=None, plugins=None): self._open_datasets = weakref.WeakValueDictionary({}) - directory = pathlib.Path(directory) #.resolve() + directory = pathlib.Path(directory).absolute() #.resolve() if directory.suffix != ".exdir": directory = directory.with_suffix(directory.suffix + ".exdir") self.user_mode = mode = mode or 'a' diff --git a/exdir/core/links.py b/exdir/core/links.py index 931cea1..dbe0312 100644 --- a/exdir/core/links.py +++ b/exdir/core/links.py @@ -1,5 +1,12 @@ +try: + import pathlib +except ImportError as e: + try: + import pathlib2 as pathlib + except ImportError: + raise e from . import exdir_file -from .exdir_object import Object +from .exdir_object import Object, is_nonraw_object_directory from .constants import * @@ -45,8 +52,6 @@ def __init__(self, filename, path): path=path ) self.filename = filename - with exdir_file.File(self.filename) as f: - pass @property def _link(self): diff --git a/tests/test_links.py b/tests/test_links.py index da66ad9..c0b9386 100644 --- a/tests/test_links.py +++ b/tests/test_links.py @@ -96,32 +96,29 @@ def test_exc(setup_teardown_file): # Feature: Create and manage external links def test_external_path(setup_teardown_file): """ External link paths attributes """ - g = File(setup_teardown_file[0] / 'foo.exdir', 'w') + external_path = setup_teardown_file[0] / 'foo.exdir' + g = File(external_path, 'w') egrp = g.create_group('foo') - el = ExternalLink(setup_teardown_file[0] / 'foo.exdir', '/foo') - assert el.filename == 'foo.exdir' + el = ExternalLink(external_path, '/foo') + assert el.filename == external_path assert el.path == '/foo' -def test_external_must_exist(setup_teardown_file): - """ External link paths attributes """ - with pytest.raises(FileExistsError): - el = ExternalLink('foo.exdir', '/foo') - - def test_external_repr(setup_teardown_file): """ External link repr """ - g = File(setup_teardown_file[0] / 'foo.exdir', 'w') - el = ExternalLink(setup_teardown_file[0] / 'foo.exdir', '/foo') + external_path = setup_teardown_file[0] / 'foo.exdir' + g = File(external_path, 'w') + el = ExternalLink(external_path, '/foo') assert isinstance(repr(el), str) def test_create(setup_teardown_file): """ Creating external links """ + external_path = setup_teardown_file[0] / 'foo.exdir' f = setup_teardown_file[3] - g = File(setup_teardown_file[0] / 'foo.exdir', 'w') + g = File(external_path, 'w') egrp = g.require_group('external') - f['ext'] = ExternalLink(setup_teardown_file[0] / 'foo.exdir', '/external') + f['ext'] = ExternalLink(external_path, '/external') grp = f['ext'] ef = grp.file assert ef != f @@ -130,63 +127,64 @@ def test_create(setup_teardown_file): def test_broken_external_link(setup_teardown_file): """ KeyError raised when attempting to open broken link """ + external_path = setup_teardown_file[0] / 'foo.exdir' f = setup_teardown_file[3] - g = File(setup_teardown_file[0] / 'foo.exdir', 'w') - f['ext'] = ExternalLink(setup_teardown_file[0] / 'foo.exdir', '/missing') + g = File(external_path, 'w') + f['ext'] = ExternalLink(external_path, '/missing') with pytest.raises(KeyError): f['ext'] -# -# # I would prefer IOError but there's no way to fix this as the exception -# # class is determined by HDF5. -# def test_exc_missingfile(setup_teardown_file): -# """ KeyError raised when attempting to open missing file """ -# f['ext'] = ExternalLink('mongoose.exdir','/foo') -# with assertRaises(KeyError): -# f['ext'] -# -# def test_close_file(setup_teardown_file): -# """ Files opened by accessing external links can be closed -# Issue 189. -# """ -# f['ext'] = ExternalLink(ename, '/') -# grp = f['ext'] -# f2 = grp.file -# f2.close() -# assertFalse(f2) -# -# + + +def test_exc_missingfile(setup_teardown_file): + """ KeyError raised when attempting to open missing file """ + f = setup_teardown_file[3] + f['ext'] = ExternalLink('mongoose.exdir','/foo') + with pytest.raises(RuntimeError): + f['ext'] + + +def test_close_file(setup_teardown_file): + """ Files opened by accessing external links can be closed + """ + external_path = setup_teardown_file[0] / 'foo.exdir' + f = setup_teardown_file[3] + g = File(external_path, 'w') + f['ext'] = ExternalLink(external_path, '/') + grp = f['ext'] + f2 = grp.file + f2.close() + assert not f2 + +# TODO uncomment if we start accepting unicode names # def test_unicode_encode(setup_teardown_file): # """ # Check that external links encode unicode filenames properly -# Testing issue #732 # """ -# ext_filename = os.path.join(mkdtemp(), u"α.exdir") -# with File(ext_filename, "w") as ext_file: +# external_path = setup_teardown_file[0] / u"α.exdir" +# with File(external_path, "w") as ext_file: # ext_file.create_group('external') -# f['ext'] = ExternalLink(ext_filename, '/external') +# f['ext'] = ExternalLink(external_path, '/external') # # # def test_unicode_decode(setup_teardown_file): # """ # Check that external links decode unicode filenames properly -# Testing issue #732 # """ -# ext_filename = os.path.join(mkdtemp(), u"α.exdir") -# with File(ext_filename, "w") as ext_file: +# external_path = setup_teardown_file[0] / u"α.exdir" +# with File(external_path, "w") as ext_file: # ext_file.create_group('external') # ext_file["external"].attrs["ext_attr"] = "test" -# f['ext'] = ExternalLink(ext_filename, '/external') -# assertEqual(f["ext"].attrs["ext_attr"], "test") +# f['ext'] = ExternalLink(external_path, '/external') +# assert f["ext"].attrs["ext_attr"] == "test" # # # def test_unicode_exdir_path(setup_teardown_file): # """ # Check that external links handle unicode exdir paths properly -# Testing issue #333 # """ -# ext_filename = os.path.join(mkdtemp(), "external.exdir") -# with File(ext_filename, "w") as ext_file: +# external_path = setup_teardown_file[0] / u"external.exdir" +# with File(external_path, "w") as ext_file: # ext_file.create_group(u'α') # ext_file[u"α"].attrs["ext_attr"] = "test" -# f['ext'] = ExternalLink(ext_filename, u'/α') +# f['ext'] = ExternalLink(external_path, u'/α') # assertEqual(f["ext"].attrs["ext_attr"], "test") From b2a1f901f1a17d0fc74198d6a4573ee1342aab55 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mikkel=20Elle=20Lepper=C3=B8d?= Date: Mon, 13 May 2019 17:24:00 +0200 Subject: [PATCH 5/8] Support for pandas DataFrames with pyarrow.feather --- exdir/core/dataset.py | 104 ++++--- exdir/core/exdir_file.py | 4 +- exdir/core/group.py | 79 ++--- tests/test_dataframe.py | 608 +++++++++++++++++++++++++++++++++++++++ 4 files changed, 727 insertions(+), 68 deletions(-) create mode 100644 tests/test_dataframe.py diff --git a/exdir/core/dataset.py b/exdir/core/dataset.py index 9fe36fd..6435283 100644 --- a/exdir/core/dataset.py +++ b/exdir/core/dataset.py @@ -1,10 +1,16 @@ import numbers import numpy as np +import pyarrow.feather as feather +import pandas as pd import exdir from . import exdir_object as exob from .mode import assert_file_open, OpenMode, assert_file_writable +NUMPY_SUFFIX = '.npy' +FEATHER_SUFFIX = '.feather' + + def _prepare_write(data, plugins, attrs, meta): for plugin in plugins: dataset_data = exdir.plugin_interface.DatasetData( @@ -25,7 +31,12 @@ def _prepare_write(data, plugins, attrs, meta): def _dataset_filename(dataset_directory): - return dataset_directory / "data.npy" + base = dataset_directory / "data" + if base.with_suffix(FEATHER_SUFFIX).exists(): + filename = base.with_suffix(FEATHER_SUFFIX) + else: + filename = base.with_suffix(NUMPY_SUFFIX) + return filename class Dataset(exob.Object): @@ -44,9 +55,8 @@ def __init__(self, root_directory, parent_path, object_name, file): object_name=object_name, file=file ) - self._data_memmap = None + self._data_loaded = None self.plugin_manager = file.plugin_manager - self.data_filename = str(_dataset_filename(self.directory)) def __getitem__(self, args): assert_file_open(self.file) @@ -75,9 +85,8 @@ def __getitem__(self, args): meta = self.meta.to_dict() atts = self.attrs.to_dict() - dataset_data = exdir.plugin_interface.DatasetData(data=values, - attrs=self.attrs.to_dict(), - meta=meta) + dataset_data = exdir.plugin_interface.DatasetData( + data=values, attrs=self.attrs.to_dict(), meta=meta) for plugin in plugins: dataset_data = plugin.prepare_read(dataset_data) @@ -100,8 +109,10 @@ def __setitem__(self, args, value): def _reload_data(self): assert_file_open(self.file) + data_filename = _dataset_filename(self.directory) for plugin in self.plugin_manager.dataset_plugins.write_order: - plugin.before_load(self.data_filename) + plugin.before_load(str(data_filename)) + if self.file.io_mode == OpenMode.READ_ONLY: mmap_mode = "r" @@ -109,35 +120,51 @@ def _reload_data(self): mmap_mode = "r+" try: - self._data_memmap = np.load(self.data_filename, mmap_mode=mmap_mode, allow_pickle=False) + if data_filename.suffix == NUMPY_SUFFIX: + self._data_loaded = np.load( + str(data_filename), + mmap_mode=mmap_mode, allow_pickle=False) + else: + self._data_loaded = feather.read_feather(str(data_filename)) self.file._open_datasets[self.name] = self except ValueError as e: - # Could be that it is a Git LFS file. Let's see if that is the case and warn if so. - with open(self.data_filename, "r") as f: + # Could be that it is a Git LFS file. + # Let's see if that is the case and warn if so. + with open(str(data_filename), "r") as f: test_string = "version https://git-lfs.github.com/spec/v1" contents = f.read(len(test_string)) if contents == test_string: raise IOError("The file '{}' is a Git LFS placeholder. " - "Open the the Exdir File with the Git LFS plugin or run " - "`git lfs fetch` first. ".format(self.data_filename)) + "Open the the Exdir File with the Git LFS plugin or run" + " `git lfs fetch` first. ".format(str(data_filename))) else: raise e def _reset_data(self, value, attrs, meta): assert_file_open(self.file) - self._data_memmap = np.lib.format.open_memmap( - self.data_filename, - mode="w+", - dtype=value.dtype, - shape=value.shape - ) - - if len(value.shape) == 0: - # scalars need to be set with itemset - self._data_memmap.itemset(value) + data_filename = _dataset_filename(self.directory) + if isinstance(value, pd.DataFrame): + feather.write_feather( + value, str(data_filename.with_suffix(FEATHER_SUFFIX))) + if data_filename.with_suffix(NUMPY_SUFFIX).exists(): + data_filename.with_suffix(NUMPY_SUFFIX).unlink() else: - # replace the contents with the value - self._data_memmap[:] = value + self._data_loaded = np.lib.format.open_memmap( + str(data_filename.with_suffix(NUMPY_SUFFIX)), + mode="w+", + dtype=value.dtype, + shape=value.shape + ) + + if len(value.shape) == 0: + # scalars need to be set with itemset + self._data_loaded.itemset(value) + else: + # replace the contents with the value + self._data_loaded[:] = value + + if data_filename.with_suffix(FEATHER_SUFFIX).exists(): + data_filename.with_suffix(FEATHER_SUFFIX).unlink() # update attributes and plugin metadata if attrs: @@ -177,7 +204,7 @@ def data(self): @data.setter def data(self, value): assert_file_open(self.file) - if self._data.shape != value.shape or self._data.dtype != value.dtype: + if isinstance(value, pd.DataFrame): value, attrs, meta = _prepare_write( data=value, plugins=self.plugin_manager.dataset_plugins.write_order, @@ -185,9 +212,22 @@ def data(self, value): meta=self.meta.to_dict() ) self._reset_data(value, attrs, meta) - return - - self[:] = value + else: + if hasattr(self._data, 'dtype'): + new_dtype = self._data.dtype != value.dtype + else: + new_dtype = True # changing from feather to numpy + if self._data.shape != value.shape or new_dtype: + value, attrs, meta = _prepare_write( + data=value, + plugins=self.plugin_manager.dataset_plugins.write_order, + attrs=self.attrs.to_dict(), + meta=self.meta.to_dict() + ) + self._reset_data(value, attrs, meta) + return + + self[:] = value @property def shape(self): @@ -270,12 +310,12 @@ def __str__(self): def __repr__(self): if self.file.io_mode == OpenMode.FILE_CLOSED: return "" - return "".format( - self.name, self.shape, self.dtype) + return "".format( + self.name, self.shape) @property def _data(self): assert_file_open(self.file) - if self._data_memmap is None: + if self._data_loaded is None: self._reload_data() - return self._data_memmap + return self._data_loaded diff --git a/exdir/core/exdir_file.py b/exdir/core/exdir_file.py index 097e29b..cec16e3 100644 --- a/exdir/core/exdir_file.py +++ b/exdir/core/exdir_file.py @@ -170,8 +170,8 @@ def close(self): # there are no way to close the memmap other than deleting all # references to it, thus try: - data_set._data_memmap.flush() - data_set._data_memmap.setflags(write=False) # TODO does not work + data_set._data_loaded.flush() + data_set._data_loaded.setflags(write=False) # TODO does not work except AttributeError: pass # force garbage collection to clean weakrefs diff --git a/exdir/core/group.py b/exdir/core/group.py index 106895f..f75fc17 100644 --- a/exdir/core/group.py +++ b/exdir/core/group.py @@ -9,6 +9,7 @@ raise e import numpy as np import exdir +import pandas as pd try: import ruamel_yaml as yaml except ImportError: @@ -129,22 +130,33 @@ def create_dataset(self, name, shape=None, dtype=None, meta=exob._default_metadata(exob.DATASET_TYPENAME) ) - _assert_data_shape_dtype_match(prepared_data, shape, dtype) + if not isinstance(data, pd.DataFrame): - shape, dtype = _data_to_shape_and_dtype(prepared_data, shape, dtype) + _assert_data_shape_dtype_match(prepared_data, shape, dtype) - if prepared_data is not None: - if shape is not None and prepared_data.shape != shape: - prepared_data = np.reshape(prepared_data, shape) - else: - if shape is None: - prepared_data = None - else: - fillvalue = fillvalue or 0.0 - prepared_data = np.full(shape, fillvalue, dtype=dtype) + shape, dtype = _data_to_shape_and_dtype(prepared_data, shape, dtype) - if prepared_data is None: - raise TypeError("Could not create a meaningful dataset.") + if prepared_data is not None: + if shape is not None and prepared_data.shape != shape: + prepared_data = np.reshape(prepared_data, shape) + else: + if shape is None: + prepared_data = None + else: + fillvalue = fillvalue or 0.0 + prepared_data = np.full(shape, fillvalue, dtype=dtype) + + if prepared_data is None: + raise TypeError("Could not create a meaningful dataset.") + else: + if dtype is not None: + raise NotImplementedError( + 'We currently do not support forcing dtype on creating with' + ' DataFrames.') + if shape is not None: + raise NotImplementedError( + 'We currently do not support reshape on creating with ' + 'DataFrames.') dataset_directory = self.directory / name exob._create_object_directory(dataset_directory, meta) @@ -311,30 +323,30 @@ def require_dataset(self, name, shape=None, dtype=None, exact=False, # TODO verify proper attributes + if not isinstance(data, pd.DataFrame): + _assert_data_shape_dtype_match(data, shape, dtype) + shape, dtype = _data_to_shape_and_dtype(data, shape, dtype) - _assert_data_shape_dtype_match(data, shape, dtype) - shape, dtype = _data_to_shape_and_dtype(data, shape, dtype) - - if not np.array_equal(shape, current_object.shape): - raise TypeError( - "Shapes do not match (existing {} vs " - "new {})".format(current_object.shape, shape) - ) - - if dtype != current_object.dtype: - if exact: + if not np.array_equal(shape, current_object.shape): raise TypeError( - "Datatypes do not exactly match " - "existing {} vs new {})".format(current_object.dtype, dtype) + "Shapes do not match (existing {} vs " + "new {})".format(current_object.shape, shape) ) - if not np.can_cast(dtype, current_object.dtype): - raise TypeError( - "Cannot safely cast from {} to {}".format( - dtype, - current_object.dtype + if dtype != current_object.dtype: + if exact: + raise TypeError( + "Datatypes do not exactly match " + "existing {} vs new {})".format(current_object.dtype, dtype) + ) + + if not np.can_cast(dtype, current_object.dtype): + raise TypeError( + "Cannot safely cast from {} to {}".format( + dtype, + current_object.dtype + ) ) - ) return current_object @@ -480,8 +492,7 @@ def __setitem__(self, name, value): raise RuntimeError( "Unable to assign value, {} already exists".format(name) ) - - self[name].value = value + self[name].data = value def __delitem__(self, name): """ diff --git a/tests/test_dataframe.py b/tests/test_dataframe.py new file mode 100644 index 0000000..eed5529 --- /dev/null +++ b/tests/test_dataframe.py @@ -0,0 +1,608 @@ +# -*- coding: utf-8 -*- + +# This file is part of Exdir, the Experimental Directory Structure. +# +# Copyright 2017 Simen Tennøe +# +# License: MIT, see "LICENSE" file for the full license terms. +# +# This file contains code from h5py, a Python interface to the HDF5 library, +# licensed under a standard 3-clause BSD license +# with copyright Andrew Collette and contributors. +# See http://www.h5py.org and the "3rdparty/h5py-LICENSE" file for details. + + +import pytest +import numpy as np +import pandas as pd +import os + +from exdir.core import Attribute, File, Dataset + +# TODO add the code below for testing true equality when parallelizing +# def __eq__(self, other): +# self[:] +# if isinstance(other, self.__class__): +# other[:] +# if self.__dict__.keys() != other.__dict__.keys(): +# return False +# +# for key in self.__dict__: +# if key == "_data": +# if not np.array_equal(self.__dict__["_data"], other.__dict__["_data"]): +# return False +# else: +# if self.__dict__[key] != other.__dict__[key]: +# return False +# return True +# else: +# return False + + +# Feature: Datasets can be created from a shape only + +def test_create_empty(setup_teardown_file): + """Create a scalar dataset.""" + f = setup_teardown_file[3] + grp = f.create_group("test") + data = pd.DataFrame([]) + dset = grp.create_dataset('foo', data=data) + assert dset.shape == (0,0) + assert dset.data.equals(data) + + +def test_create_scalar(setup_teardown_file): + """Create a size-1 dataset.""" + f = setup_teardown_file[3] + grp = f.create_group("test") + data = pd.DataFrame([1]) + dset = grp.create_dataset('foo', data=data) + assert dset.shape == (1,1) + assert dset.shape == dset.data.shape + assert data.values == dset.data.values + + +def test_create_extended(setup_teardown_file): + """Create an extended dataset.""" + f = setup_teardown_file[3] + grp = f.create_group("test") + data = pd.DataFrame(np.arange(63)) + dset = grp.create_dataset('foo', data=data) + assert dset.shape == (63,1) + assert dset.size == 63 + + data = pd.DataFrame(np.zeros((6,10))) + dset = f.create_dataset('bar', data=data) + assert dset.shape == (6, 10) + assert dset.size == (60) + + +def test_no_dtype(setup_teardown_file): + """Confirm that datafram has no dtype.""" + f = setup_teardown_file[3] + grp = f.create_group("test") + + data = pd.DataFrame(np.zeros((6,10))) + dset = f.create_dataset('bar', data=data) + with pytest.raises(AttributeError): + dset.dtype + + +def test_no_dtype_create(setup_teardown_file): + """Confirm that one can force dtype """ + f = setup_teardown_file[3] + data = pd.DataFrame(np.zeros((6,10))) + with pytest.raises(NotImplementedError): + f.create_dataset('bar', data=data, dtype=np.int16) + + +def test_numpy_then_dataframe(setup_teardown_file): + """Confirm that datafram has no dtype.""" + f = setup_teardown_file[3] + grp = f.create_group("test") + + data = pd.DataFrame(np.zeros((6,10))) + dset = f.create_dataset('bar', (6,10)) + assert isinstance(f['bar'].data, np.ndarray) + f['bar'] = data + assert np.array_equal(f['bar'].data.values, data.values) + assert isinstance(f['bar'].data, pd.DataFrame) + assert not (dset.directory / 'data.npy').exists() + + +def test_datafram_then_numpy(setup_teardown_file): + """Confirm that datafram has no dtype.""" + f = setup_teardown_file[3] + grp = f.create_group("test") + + data = pd.DataFrame(np.zeros((6,10))) + dset = f.create_dataset('bar', data=data) + assert isinstance(f['bar'].data, pd.DataFrame) + f['bar'] = np.zeros((6,10)) + assert np.array_equal(f['bar'].data, np.zeros((6,10))) + assert isinstance(f['bar'].data, np.ndarray) + assert not (dset.directory / 'data.feather').exists() + + +def test_reshape(setup_teardown_file): + """Create from existing data, and make it fit a new shape.""" + f = setup_teardown_file[3] + grp = f.create_group("test") + + data = pd.DataFrame(np.zeros((6,10))) + with pytest.raises(NotImplementedError): + dset = grp.create_dataset('foo', shape=(10, 3), data=data) + +# # Feature: Datasets can be created only if they don't exist in the file +def test_create(setup_teardown_file): + """Create new dataset with no conflicts.""" + f = setup_teardown_file[3] + grp = f.create_group("test") + + data = pd.DataFrame(np.zeros((10, 3))) + dset = grp.require_dataset('foo', (10, 3)) + assert isinstance(dset, Dataset) + assert dset.shape == (10, 3) + + +def test_create_existing(setup_teardown_file): + """require_dataset yields existing dataset.""" + f = setup_teardown_file[3] + grp = f.create_group("test") + + data2 = pd.DataFrame(np.zeros((3, 10))) + data3 = pd.DataFrame(np.zeros((4, 11))) + dset2 = grp.require_dataset('bar', data=data2) + dset3 = grp.require_dataset('bar', data=data3) + assert isinstance(dset2, Dataset) + assert np.array_equal(dset2.data.values, data2.values) + assert np.array_equal(dset3.data.values, data2.values) + assert dset2 == dset3 +# +# +# def test_shape_conflict(setup_teardown_file): +# """require_dataset with shape conflict yields TypeError.""" +# f = setup_teardown_file[3] +# grp = f.create_group("test") +# +# grp.create_dataset('foo', (10, 3), 'f') +# with pytest.raises(TypeError): +# grp.require_dataset('foo', (10, 4), 'f') +# +# +# def test_type_confict(setup_teardown_file): +# """require_dataset with object type conflict yields TypeError.""" +# f = setup_teardown_file[3] +# grp = f.create_group("test") +# +# grp.create_group('foo') +# with pytest.raises(TypeError): +# grp.require_dataset('foo', (10, 3), 'f') +# +# +# def test_dtype_conflict(setup_teardown_file): +# """require_dataset with dtype conflict (strict mode) yields TypeError.""" +# f = setup_teardown_file[3] +# grp = f.create_group("test") +# +# dset = grp.create_dataset('foo', (10, 3), 'f') +# with pytest.raises(TypeError): +# grp.require_dataset('foo', (10, 3), 'S10') +# +# +# def test_dtype_close(setup_teardown_file): +# """require_dataset with convertible type succeeds (non-strict mode)-""" +# f = setup_teardown_file[3] +# grp = f.create_group("test") +# +# dset = grp.create_dataset('foo', (10, 3), 'i4') +# dset2 = grp.require_dataset('foo', (10, 3), 'i2', exact=False) +# assert dset == dset2 +# assert dset2.dtype == np.dtype('i4') +# +# +# # Feature: Datasets can be created with fill value +# +# def test_create_fillval(setup_teardown_file): +# """Fill value is reflected in dataset contents.""" +# f = setup_teardown_file[3] +# grp = f.create_group("test") +# +# dset = grp.create_dataset('foo', (10,), fillvalue=4.0) +# assert dset[0] == 4.0 +# assert dset[7] == 4.0 +# +# +# +# def test_compound(setup_teardown_file): +# """Fill value works with compound types.""" +# f = setup_teardown_file[3] +# grp = f.create_group("test") +# +# dt = np.dtype([('a', 'f4'), ('b', 'i8')]) +# v = np.ones((1,), dtype=dt)[0] +# dset = grp.create_dataset('foo', (10,), dtype=dt, fillvalue=v) +# +# +# def test_exc(setup_teardown_file): +# """Bogus fill value raises TypeError.""" +# f = setup_teardown_file[3] +# grp = f.create_group("test") +# +# with pytest.raises(TypeError): +# grp.create_dataset('foo', (10,), dtype="float32", fillvalue={"a": 2}) +# +# +# def test_string(setup_teardown_file): +# """Assignement of fixed-length byte string produces a fixed-length +# ascii dataset """ +# f = setup_teardown_file[3] +# grp = f.create_group("test") +# +# dset = grp.create_dataset('foo', data="string") +# assert dset.data == "string" +# +# +# +# # Feature: Dataset dtype is available as .dtype property +# +# def test_dtype(setup_teardown_file): +# """Retrieve dtype from dataset.""" +# f = setup_teardown_file[3] +# grp = f.create_group("test") +# +# dset = grp.create_dataset('foo', (5,), '|S10') +# assert dset.dtype == np.dtype('|S10') +# +# +# # Feature: Size of first axis is available via Python's len +# def test_len(setup_teardown_file): +# """len().""" +# f = setup_teardown_file[3] +# grp = f.create_group("test") +# +# dset = grp.create_dataset('foo', (312, 15)) +# assert len(dset) == 312 +# +# +# def test_len_scalar(setup_teardown_file): +# """len() of scalar).""" +# f = setup_teardown_file[3] +# grp = f.create_group("test") +# +# dset =grp.create_dataset('foo', data=1) +# with pytest.raises(TypeError): +# len(dset) +# +# +# # Feature: Iterating over a dataset yields rows +# +# def test_iter(setup_teardown_file): +# """Iterating over a dataset yields rows.""" +# f = setup_teardown_file[3] +# grp = f.create_group("test") +# +# data = np.arange(30, dtype='f').reshape((10, 3)) +# dset = grp.create_dataset('foo', data=data) +# for x, y in zip(dset, data): +# assert len(x) == 3 +# assert np.array_equal(x, y) +# +# +# def test_iter_scalar(setup_teardown_file): +# """Iterating over scalar dataset raises TypeError.""" +# f = setup_teardown_file[3] +# grp = f.create_group("test") +# +# dset = grp.create_dataset('foo', shape=()) +# with pytest.raises(TypeError): +# [x for x in dset] +# +# +# def test_trailing_slash(setup_teardown_file): +# """Trailing slashes are unconditionally ignored.""" +# f = setup_teardown_file[3] +# +# f["dataset"] = 42 +# assert "dataset/" in f +# +# +# # Feature: Compound types correctly round-trip +# def test_compund(setup_teardown_file): +# """Compound types are read back in correct order.""" +# f = setup_teardown_file[3] +# grp = f.create_group("test") +# +# dt = np.dtype( [('weight', np.float64), +# ('cputime', np.float64), +# ('walltime', np.float64), +# ('parents_offset', np.uint32), +# ('n_parents', np.uint32), +# ('status', np.uint8), +# ('endpoint_type', np.uint8)]) +# +# testdata = np.ndarray((16,), dtype=dt) +# for key in dt.fields: +# testdata[key] = np.random.random((16,))*100 +# +# # print(testdata) +# +# grp['test'] = testdata +# outdata = grp['test'][()] +# assert np.all(outdata == testdata) +# assert outdata.dtype == testdata.dtype +# +# def test_assign(setup_teardown_file): +# f = setup_teardown_file[3] +# grp = f.create_group("test") +# +# dt = np.dtype([('weight', (np.float64, 3)), +# ('endpoint_type', np.uint8),]) +# +# testdata = np.ndarray((16,), dtype=dt) +# for key in dt.fields: +# testdata[key] = np.random.random(size=testdata[key].shape)*100 +# +# ds = grp.create_dataset('test', (16,), dtype=dt) +# for key in dt.fields: +# ds[key] = testdata[key] +# +# outdata = f['test']["test"][()] +# +# assert np.all(outdata == testdata) +# assert outdata.dtype == testdata.dtype +# +# +# +# +# def test_set_data(setup_teardown_file): +# """Set data works correctly.""" +# f = setup_teardown_file[3] +# grp = f.create_group("test") +# +# testdata = np.ones((10, 2)) +# grp['testdata'] = testdata +# outdata = grp['testdata'][()] +# assert np.all(outdata == testdata) +# assert outdata.dtype == testdata.dtype +# +# grp['testdata'] = testdata +# +# +# +# +# def test_eq_false(setup_teardown_file): +# """__eq__.""" +# f = setup_teardown_file[3] +# grp = f.create_group("test") +# +# dset = grp.create_dataset('foo', data=1) +# dset2 = grp.create_dataset('foobar', (2, 2)) +# +# assert dset != dset2 +# assert not dset == 2 +# +# def test_eq(setup_teardown_file): +# """__eq__.""" +# f = setup_teardown_file[3] +# grp = f.create_group("test") +# +# dset = grp.create_dataset('foo', data=np.ones((2, 2))) +# +# assert dset == dset +# +# +# def test_mmap(setup_teardown_file): +# """Test that changes to a mmap loaded numpy file is written to disk""" +# f = setup_teardown_file[3] +# grp = f.create_group("test") +# +# dset = grp.create_dataset('foo', (10**3, 10**3), fillvalue=2) +# dset[1, 1] = 100 +# +# tmp_file = np.load(str(setup_teardown_file[1] / "test" / "foo" / "data.npy")) +# +# assert dset.data[1, 1] == 100 +# assert tmp_file[1, 1] == 100 +# +# +# def test_modify_view(setup_teardown_file): +# f = setup_teardown_file[3] +# dataset = f.create_dataset("mydata", data=np.array([1, 2, 3, 4, 5, 6, 7, 8])) +# dataset[3:5] = np.array([8, 9]) +# assert np.array_equal(f["mydata"][3:5], np.array([8, 9])) +# view = dataset[3:5] +# view[0] = 10 +# assert f["mydata"][3] == 10 +# +# +# def test_single_index(setup_teardown_file): +# """Single-element selection with [index] yields array scalar.""" +# f = setup_teardown_file[3] +# dset = f.create_dataset('x', (1,), dtype='i1') +# out = dset[0] +# assert isinstance(out, np.int8) +# +# def test_single_null(setup_teardown_file): +# """Single-element selection with [()] yields ndarray.""" +# f = setup_teardown_file[3] +# +# dset = f.create_dataset('x', (1,), dtype='i1') +# out = dset[()] +# assert isinstance(out, np.ndarray) +# assert out.shape == (1,) +# +# def test_scalar_index(setup_teardown_file): +# """Slicing with [...] yields scalar ndarray.""" +# f = setup_teardown_file[3] +# +# dset = f.create_dataset('x', shape=(), dtype='f') +# out = dset[...] +# assert isinstance(out, np.ndarray) +# assert out.shape == () +# +# def test_scalar_null(setup_teardown_file): +# """Slicing with [()] yields array scalar.""" +# f = setup_teardown_file[3] +# +# dset = f.create_dataset('x', shape=(), dtype='i1') +# out = dset[()] +# +# assert out.dtype == "int8" +# +# def test_compound_index(setup_teardown_file): +# """Compound scalar is numpy.void, not tuple.""" +# f = setup_teardown_file[3] +# +# dt = np.dtype([('a', 'i4'), ('b', 'f8')]) +# v = np.ones((4,), dtype=dt) +# dset = f.create_dataset('foo', (4,), data=v) +# assert dset[0] == v[0] +# assert isinstance(dset[0], np.void) +# +# +# # Feature: Simple NumPy-style slices (start:stop:step) are supported. +# +# def test_negative_stop(setup_teardown_file): +# """Negative stop indexes work as they do in NumPy.""" +# f = setup_teardown_file[3] +# +# arr = np.arange(10) +# dset = f.create_dataset('x', data=arr) +# +# assert np.array_equal(dset[2:-2], arr[2:-2]) +# +# +# # Feature: Array types are handled appropriately +# +# def test_read(setup_teardown_file): +# """Read arrays tack array dimensions onto end of shape tuple.""" +# f = setup_teardown_file[3] +# +# dt = np.dtype('(3,)f8') +# dset = f.create_dataset('x', (10,), dtype=dt) +# # TODO implement this +# # assert dset.shape == (10,) +# # assert dset.dtype == dt +# +# # Full read +# out = dset[...] +# assert out.dtype == np.dtype('f8') +# assert out.shape == (10, 3) +# +# # Single element +# out = dset[0] +# assert out.dtype == np.dtype('f8') +# assert out.shape == (3,) +# +# # Range +# out = dset[2:8:2] +# assert out.dtype == np.dtype('f8') +# assert out.shape == (3, 3) +# +# def test_write_broadcast(setup_teardown_file): +# """Array fill from constant is supported.""" +# f = setup_teardown_file[3] +# +# dt = np.dtype('(3,)i') +# +# dset = f.create_dataset('x', (10,), dtype=dt) +# dset[...] = 42 +# +# +# +# def test_write_element(setup_teardown_file): +# """Write a single element to the array.""" +# f = setup_teardown_file[3] +# +# dt = np.dtype('(3,)f8') +# dset = f.create_dataset('x', (10,), dtype=dt) +# +# data = np.array([1, 2, 3.0]) +# dset[4] = data +# +# out = dset[4] +# assert np.all(out == data) +# +# +# def test_write_slices(setup_teardown_file): +# """Write slices to array type.""" +# f = setup_teardown_file[3] +# +# dt = np.dtype('(3,)i') +# +# data1 = np.ones((2, ), dtype=dt) +# data2 = np.ones((4, 5), dtype=dt) +# +# dset = f.create_dataset('x', (10, 9, 11), dtype=dt) +# +# dset[0, 0, 2:4] = data1 +# assert np.array_equal(dset[0, 0, 2:4], data1) +# +# dset[3, 1:5, 6:11] = data2 +# assert np.array_equal(dset[3, 1:5, 6:11], data2) +# +# +# def test_roundtrip(setup_teardown_file): +# """Read the contents of an array and write them back.""" +# f = setup_teardown_file[3] +# dt = np.dtype('(3,)f8') +# dset = f.create_dataset('x', (10,), dtype=dt) +# +# out = dset[...] +# dset[...] = out +# +# assert np.all(dset[...] == out) +# +# +# +# # Feature Slices resulting in empty arrays +# +# +# def test_slice_zero_length_dimension(setup_teardown_file): +# """Slice a dataset with a zero in its shape vector +# along the zero-length dimension.""" +# f = setup_teardown_file[3] +# +# for i, shape in enumerate([(0,), (0, 3), (0, 2, 1)]): +# dset = f.create_dataset('x%d'%i, shape, dtype=np.int) +# assert dset.shape == shape +# out = dset[...] +# assert isinstance(out, np.ndarray) +# assert out.shape == shape +# out = dset[:] +# assert isinstance(out, np.ndarray) +# assert out.shape == shape +# if len(shape) > 1: +# out = dset[:, :1] +# assert isinstance(out, np.ndarray) +# assert out.shape[:2] == (0, 1) +# +# def test_slice_other_dimension(setup_teardown_file): +# """Slice a dataset with a zero in its shape vector +# along a non-zero-length dimension.""" +# f = setup_teardown_file[3] +# +# for i, shape in enumerate([(3, 0), (1, 2, 0), (2, 0, 1)]): +# dset = f.create_dataset('x%d'%i, shape, dtype=np.int) +# assert dset.shape == shape +# out = dset[:1] +# assert isinstance(out, np.ndarray) +# assert out.shape == (1,)+shape[1:] +# +# def test_slice_of_length_zero(setup_teardown_file): +# """Get a slice of length zero from a non-empty dataset.""" +# f = setup_teardown_file[3] +# +# for i, shape in enumerate([(3, ), (2, 2, ), (2, 1, 5)]): +# dset = f.create_dataset('x%d'%i, data=np.zeros(shape, np.int)) +# assert dset.shape == shape +# out = dset[1:1] +# assert isinstance(out, np.ndarray) +# assert out.shape == (0,)+shape[1:] +# +# def test_modify_all(setup_teardown_file): +# f = setup_teardown_file[3] +# dset = f.create_dataset("test", data=np.arange(10)) +# dset.data = np.ones(4) +# assert np.all(dset.data == np.ones(4)) From e12b9cc473824754e66e7505a474e292549c22cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mikkel=20E=20Lepper=C3=B8d?= Date: Tue, 14 May 2019 06:18:48 +0200 Subject: [PATCH 6/8] import pandas on ci build --- .conda-recipe/meta.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.conda-recipe/meta.yaml b/.conda-recipe/meta.yaml index 83678be..f8c32f4 100644 --- a/.conda-recipe/meta.yaml +++ b/.conda-recipe/meta.yaml @@ -23,6 +23,7 @@ requirements: - numpy - scipy - ruamel_yaml + - pandas - pyyaml - pathlib # [py2k] - enum34 # [py2k] @@ -34,6 +35,7 @@ test: - pytest - pytest-benchmark - h5py + - pandas - six - coverage - codecov From 8d01a05f37baa3d204bc9966113a57534f365930 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mikkel=20E=20Lepper=C3=B8d?= Date: Tue, 14 May 2019 19:59:24 +0200 Subject: [PATCH 7/8] dataframes handled properly upon require --- .conda-recipe/meta.yaml | 2 + exdir/core/dataset.py | 5 +- exdir/core/group.py | 5 ++ tests/test_dataframe.py | 194 +++++++++++++++++++++------------------- 4 files changed, 115 insertions(+), 91 deletions(-) diff --git a/.conda-recipe/meta.yaml b/.conda-recipe/meta.yaml index f8c32f4..4a75a5c 100644 --- a/.conda-recipe/meta.yaml +++ b/.conda-recipe/meta.yaml @@ -24,6 +24,7 @@ requirements: - scipy - ruamel_yaml - pandas + - pyarrow - pyyaml - pathlib # [py2k] - enum34 # [py2k] @@ -35,6 +36,7 @@ test: - pytest - pytest-benchmark - h5py + - pyarrow - pandas - six - coverage diff --git a/exdir/core/dataset.py b/exdir/core/dataset.py index 6435283..b593301 100644 --- a/exdir/core/dataset.py +++ b/exdir/core/dataset.py @@ -128,6 +128,9 @@ def _reload_data(self): self._data_loaded = feather.read_feather(str(data_filename)) self.file._open_datasets[self.name] = self except ValueError as e: + # Could be that numpy needs to pickle, suggest the user to use + # dataframe + # Could be that it is a Git LFS file. # Let's see if that is the case and warn if so. with open(str(data_filename), "r") as f: @@ -216,7 +219,7 @@ def data(self, value): if hasattr(self._data, 'dtype'): new_dtype = self._data.dtype != value.dtype else: - new_dtype = True # changing from feather to numpy + new_dtype = True # changing from feather to numpy if self._data.shape != value.shape or new_dtype: value, attrs, meta = _prepare_write( data=value, diff --git a/exdir/core/group.py b/exdir/core/group.py index f75fc17..8241cf0 100644 --- a/exdir/core/group.py +++ b/exdir/core/group.py @@ -323,6 +323,11 @@ def require_dataset(self, name, shape=None, dtype=None, exact=False, # TODO verify proper attributes + if not isinstance(data, type(current_object.data)): + raise IOError( + 'Not allowed to change data instance with "require_dataset"' + '(existing {} vs new {}), set data if a change is desired.' + ''.format(type(current_object.data), type(data))) if not isinstance(data, pd.DataFrame): _assert_data_shape_dtype_match(data, shape, dtype) shape, dtype = _data_to_shape_and_dtype(data, shape, dtype) diff --git a/tests/test_dataframe.py b/tests/test_dataframe.py index eed5529..e09cb8d 100644 --- a/tests/test_dataframe.py +++ b/tests/test_dataframe.py @@ -39,7 +39,27 @@ # return False -# Feature: Datasets can be created from a shape only +# NOTE feather converts integer column names to str +def dataframe_equal(orig_df, new_df): + columns = [] + for col1, col2 in zip(orig_df.columns, new_df.columns): + try: + columns.append(int(col2)==int(col1)) + except: + columns.append(col2==col1) + index = [] + for row1, row2 in zip(orig_df.index, new_df.index): + try: + index.append(int(row2)==int(row1)) + except: + index.append(row2==row1) + result = ( + np.array_equal(orig_df.values, new_df.values) and + all(columns) and + all(index) + ) + return result + def test_create_empty(setup_teardown_file): """Create a scalar dataset.""" @@ -59,7 +79,7 @@ def test_create_scalar(setup_teardown_file): dset = grp.create_dataset('foo', data=data) assert dset.shape == (1,1) assert dset.shape == dset.data.shape - assert data.values == dset.data.values + assert dataframe_equal(data, dset.data) def test_create_extended(setup_teardown_file): @@ -133,6 +153,7 @@ def test_reshape(setup_teardown_file): with pytest.raises(NotImplementedError): dset = grp.create_dataset('foo', shape=(10, 3), data=data) + # # Feature: Datasets can be created only if they don't exist in the file def test_create(setup_teardown_file): """Create new dataset with no conflicts.""" @@ -144,8 +165,11 @@ def test_create(setup_teardown_file): assert isinstance(dset, Dataset) assert dset.shape == (10, 3) + with pytest.raises(RuntimeError): + grp.create_dataset('foo', (10, 3)) + -def test_create_existing(setup_teardown_file): +def test_create_existing_shape_mismatch(setup_teardown_file): """require_dataset yields existing dataset.""" f = setup_teardown_file[3] grp = f.create_group("test") @@ -155,94 +179,84 @@ def test_create_existing(setup_teardown_file): dset2 = grp.require_dataset('bar', data=data2) dset3 = grp.require_dataset('bar', data=data3) assert isinstance(dset2, Dataset) - assert np.array_equal(dset2.data.values, data2.values) - assert np.array_equal(dset3.data.values, data2.values) + assert dataframe_equal(dset2.data, data2) + assert dataframe_equal(dset3.data, data2) assert dset2 == dset3 -# -# -# def test_shape_conflict(setup_teardown_file): -# """require_dataset with shape conflict yields TypeError.""" -# f = setup_teardown_file[3] -# grp = f.create_group("test") -# -# grp.create_dataset('foo', (10, 3), 'f') -# with pytest.raises(TypeError): -# grp.require_dataset('foo', (10, 4), 'f') -# -# -# def test_type_confict(setup_teardown_file): -# """require_dataset with object type conflict yields TypeError.""" -# f = setup_teardown_file[3] -# grp = f.create_group("test") -# -# grp.create_group('foo') -# with pytest.raises(TypeError): -# grp.require_dataset('foo', (10, 3), 'f') -# -# -# def test_dtype_conflict(setup_teardown_file): -# """require_dataset with dtype conflict (strict mode) yields TypeError.""" -# f = setup_teardown_file[3] -# grp = f.create_group("test") -# -# dset = grp.create_dataset('foo', (10, 3), 'f') -# with pytest.raises(TypeError): -# grp.require_dataset('foo', (10, 3), 'S10') -# -# -# def test_dtype_close(setup_teardown_file): -# """require_dataset with convertible type succeeds (non-strict mode)-""" -# f = setup_teardown_file[3] -# grp = f.create_group("test") -# -# dset = grp.create_dataset('foo', (10, 3), 'i4') -# dset2 = grp.require_dataset('foo', (10, 3), 'i2', exact=False) -# assert dset == dset2 -# assert dset2.dtype == np.dtype('i4') -# -# -# # Feature: Datasets can be created with fill value -# -# def test_create_fillval(setup_teardown_file): -# """Fill value is reflected in dataset contents.""" -# f = setup_teardown_file[3] -# grp = f.create_group("test") -# -# dset = grp.create_dataset('foo', (10,), fillvalue=4.0) -# assert dset[0] == 4.0 -# assert dset[7] == 4.0 -# -# -# -# def test_compound(setup_teardown_file): -# """Fill value works with compound types.""" -# f = setup_teardown_file[3] -# grp = f.create_group("test") -# -# dt = np.dtype([('a', 'f4'), ('b', 'i8')]) -# v = np.ones((1,), dtype=dt)[0] -# dset = grp.create_dataset('foo', (10,), dtype=dt, fillvalue=v) -# -# -# def test_exc(setup_teardown_file): -# """Bogus fill value raises TypeError.""" -# f = setup_teardown_file[3] -# grp = f.create_group("test") -# -# with pytest.raises(TypeError): -# grp.create_dataset('foo', (10,), dtype="float32", fillvalue={"a": 2}) -# -# -# def test_string(setup_teardown_file): -# """Assignement of fixed-length byte string produces a fixed-length -# ascii dataset """ -# f = setup_teardown_file[3] -# grp = f.create_group("test") -# -# dset = grp.create_dataset('foo', data="string") -# assert dset.data == "string" -# -# + + +def test_create_existing_same_shape(setup_teardown_file): + """require_dataset yields existing dataset.""" + f = setup_teardown_file[3] + grp = f.create_group("test") + + data2 = pd.DataFrame((3, 10)) + data3 = pd.DataFrame((4, 11)) + dset2 = grp.require_dataset('bar', data=data2) + dset3 = grp.require_dataset('bar', data=data3) + assert isinstance(dset2, Dataset) + assert dataframe_equal(dset2.data, data2) + assert dataframe_equal(dset3.data, data2) + assert dset2 == dset3 + + +def test_create_existing_df_to_npy(setup_teardown_file): + """require_dataset yields existing dataset.""" + f = setup_teardown_file[3] + grp = f.create_group("test") + + data2 = pd.DataFrame((3, 10)) + data3 = np.zeros((1, 2)) + dset2 = grp.require_dataset('bar', data=data2) + with pytest.raises(IOError): + grp.require_dataset('bar', data=data3) + + +def test_create_existing_npy_to_df(setup_teardown_file): + """require_dataset yields existing dataset.""" + f = setup_teardown_file[3] + grp = f.create_group("test") + + data2 = np.zeros((1, 2)) + data3 = pd.DataFrame((3, 10)) + dset2 = grp.require_dataset('bar', data=data2) + with pytest.raises(IOError): + grp.require_dataset('bar', data=data3) + + +def test_compound(setup_teardown_file): + """Fill value works with compound types.""" + f = setup_teardown_file[3] + grp = f.create_group("test") + + dt = np.dtype([('a', 'f4'), ('b', 'i8')]) + v = np.ones((1,), dtype=dt) + data = pd.DataFrame(v) + dset = grp.create_dataset('foo', data=data) + assert dataframe_equal(dset.data, data) + + +def test_variable_length_string(setup_teardown_file): + """Assignement of variable-length byte string produces a fixed-length + ascii dataset """ + f = setup_teardown_file[3] + grp = f.create_group("test") + values = np.array(['aaaa', 'aaaaaaaa']) + data = pd.DataFrame(values) + + dset = grp.create_dataset('foo', data=data) + assert dataframe_equal(dset.data, data) + + +def test_variable_length_string_numpy(setup_teardown_file): + """Assignement of variable-length byte string produces a fixed-length + ascii dataset """ + f = setup_teardown_file[3] + grp = f.create_group("test") + data = np.array(['aaaa', 'aaaaaaaa']) + # with pytest.raises(IOError): + grp.create_dataset('foo', data=data) + + # # # Feature: Dataset dtype is available as .dtype property # From c8addca6ecb418f5c9990f5c0bb0475fc39b9cb8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mikkel=20E=20Lepper=C3=B8d?= Date: Wed, 15 May 2019 11:45:46 -0400 Subject: [PATCH 8/8] properly setting dataframes and flush method --- exdir/core/dataset.py | 26 ++- exdir/core/group.py | 11 +- tests/test_dataframe.py | 442 +++++++--------------------------------- tests/test_dataset.py | 11 + 4 files changed, 106 insertions(+), 384 deletions(-) diff --git a/exdir/core/dataset.py b/exdir/core/dataset.py index b593301..78dbe18 100644 --- a/exdir/core/dataset.py +++ b/exdir/core/dataset.py @@ -34,9 +34,11 @@ def _dataset_filename(dataset_directory): base = dataset_directory / "data" if base.with_suffix(FEATHER_SUFFIX).exists(): filename = base.with_suffix(FEATHER_SUFFIX) + is_numpy = False else: filename = base.with_suffix(NUMPY_SUFFIX) - return filename + is_numpy = True + return filename, is_numpy class Dataset(exob.Object): @@ -96,6 +98,7 @@ def __getitem__(self, args): def __setitem__(self, args, value): assert_file_writable(self.file) + data_filename, is_numpy = _dataset_filename(self.directory) value, attrs, meta = _prepare_write( data=value, @@ -103,13 +106,20 @@ def __setitem__(self, args, value): attrs=self.attrs.to_dict(), meta=self.meta.to_dict() ) - self._data[args] = value + if is_numpy: + self._data[args] = value + else: + self._data[args] = value + self.flush() self.attrs = attrs self.meta._set_data(meta) + def flush(self): + self.data = self._data + def _reload_data(self): assert_file_open(self.file) - data_filename = _dataset_filename(self.directory) + data_filename, is_numpy = _dataset_filename(self.directory) for plugin in self.plugin_manager.dataset_plugins.write_order: plugin.before_load(str(data_filename)) @@ -120,7 +130,7 @@ def _reload_data(self): mmap_mode = "r+" try: - if data_filename.suffix == NUMPY_SUFFIX: + if is_numpy: self._data_loaded = np.load( str(data_filename), mmap_mode=mmap_mode, allow_pickle=False) @@ -130,7 +140,7 @@ def _reload_data(self): except ValueError as e: # Could be that numpy needs to pickle, suggest the user to use # dataframe - + # Could be that it is a Git LFS file. # Let's see if that is the case and warn if so. with open(str(data_filename), "r") as f: @@ -145,7 +155,7 @@ def _reload_data(self): def _reset_data(self, value, attrs, meta): assert_file_open(self.file) - data_filename = _dataset_filename(self.directory) + data_filename, _ = _dataset_filename(self.directory) if isinstance(value, pd.DataFrame): feather.write_feather( value, str(data_filename.with_suffix(FEATHER_SUFFIX))) @@ -304,8 +314,8 @@ def __iter__(self): if len(self.shape) == 0: raise TypeError("Can't iterate over a scalar dataset") - for i in range(self.shape[0]): - yield self[i] + for val in self.data: + yield val def __str__(self): return self.data.__str__() diff --git a/exdir/core/group.py b/exdir/core/group.py index 8241cf0..f5ecc08 100644 --- a/exdir/core/group.py +++ b/exdir/core/group.py @@ -323,11 +323,12 @@ def require_dataset(self, name, shape=None, dtype=None, exact=False, # TODO verify proper attributes - if not isinstance(data, type(current_object.data)): - raise IOError( - 'Not allowed to change data instance with "require_dataset"' - '(existing {} vs new {}), set data if a change is desired.' - ''.format(type(current_object.data), type(data))) + if any(isinstance(a, pd.DataFrame) for a in [data, current_object.data]): + if not isinstance(data, type(current_object.data)): + raise IOError( + 'Not allowed to require different data instance with' + '(existing {} vs new {}), set data if a change is desired.' + ''.format(type(current_object.data), type(data))) if not isinstance(data, pd.DataFrame): _assert_data_shape_dtype_match(data, shape, dtype) shape, dtype = _data_to_shape_and_dtype(data, shape, dtype) diff --git a/tests/test_dataframe.py b/tests/test_dataframe.py index e09cb8d..d312da7 100644 --- a/tests/test_dataframe.py +++ b/tests/test_dataframe.py @@ -235,388 +235,88 @@ def test_compound(setup_teardown_file): assert dataframe_equal(dset.data, data) -def test_variable_length_string(setup_teardown_file): +def test_variable_length_string_numpy(setup_teardown_file): """Assignement of variable-length byte string produces a fixed-length ascii dataset """ f = setup_teardown_file[3] grp = f.create_group("test") - values = np.array(['aaaa', 'aaaaaaaa']) - data = pd.DataFrame(values) + # unable to change length of string with setitem + data = np.array(['a', 'aa']) + dset = grp.create_dataset('foo', data=data) + dset[1] = 'aaaaaa' + assert np.array_equal(dset.data, data) +# sjekk at setterne faktisk endrer på fil +def test_variable_length_string_df(setup_teardown_file): + """Assignement of variable-length byte string produces a fixed-length + ascii dataset """ + f = setup_teardown_file[3] + grp = f.create_group("test") + # one needs object dtype in order to be able to change length of string with setitem + values = ['a', 'aa'] + data = pd.DataFrame(values).T dset = grp.create_dataset('foo', data=data) - assert dataframe_equal(dset.data, data) + dset['1'] = 'aaaaaa' + values[1] = 'aaaaaa' + assert np.array_equal(dset.data.values[0], values) + dset._reload_data() + assert np.array_equal(dset.data.values[0], values) -def test_variable_length_string_numpy(setup_teardown_file): + +def test_flush(setup_teardown_file): """Assignement of variable-length byte string produces a fixed-length ascii dataset """ f = setup_teardown_file[3] grp = f.create_group("test") - data = np.array(['aaaa', 'aaaaaaaa']) - # with pytest.raises(IOError): - grp.create_dataset('foo', data=data) + # one needs object dtype in order to be able to change length of string with setitem + values = ['a', 'aa'] + data = pd.DataFrame(values).T + dset = grp.create_dataset('foo', data=data) + # using iloc changes data only in memory + values[1] = 'aaaaaa' + dset.data.iloc[0,1] = 'aaaaaa' + assert np.array_equal(dset.data.values[0], values) + dset._reload_data() + assert dataframe_equal(dset.data, data) + # flush saves data to file + dset.data.iloc[0,1] = 'aaaaaa' + dset.flush() + dset._reload_data() + assert np.array_equal(dset.data.values[0], values) -# -# # Feature: Dataset dtype is available as .dtype property -# -# def test_dtype(setup_teardown_file): -# """Retrieve dtype from dataset.""" -# f = setup_teardown_file[3] -# grp = f.create_group("test") -# -# dset = grp.create_dataset('foo', (5,), '|S10') -# assert dset.dtype == np.dtype('|S10') -# -# -# # Feature: Size of first axis is available via Python's len -# def test_len(setup_teardown_file): -# """len().""" -# f = setup_teardown_file[3] -# grp = f.create_group("test") -# -# dset = grp.create_dataset('foo', (312, 15)) -# assert len(dset) == 312 -# -# -# def test_len_scalar(setup_teardown_file): -# """len() of scalar).""" -# f = setup_teardown_file[3] -# grp = f.create_group("test") -# -# dset =grp.create_dataset('foo', data=1) -# with pytest.raises(TypeError): -# len(dset) -# -# -# # Feature: Iterating over a dataset yields rows -# -# def test_iter(setup_teardown_file): -# """Iterating over a dataset yields rows.""" -# f = setup_teardown_file[3] -# grp = f.create_group("test") -# -# data = np.arange(30, dtype='f').reshape((10, 3)) -# dset = grp.create_dataset('foo', data=data) -# for x, y in zip(dset, data): -# assert len(x) == 3 -# assert np.array_equal(x, y) -# -# -# def test_iter_scalar(setup_teardown_file): -# """Iterating over scalar dataset raises TypeError.""" -# f = setup_teardown_file[3] -# grp = f.create_group("test") -# -# dset = grp.create_dataset('foo', shape=()) -# with pytest.raises(TypeError): -# [x for x in dset] -# -# -# def test_trailing_slash(setup_teardown_file): -# """Trailing slashes are unconditionally ignored.""" -# f = setup_teardown_file[3] -# -# f["dataset"] = 42 -# assert "dataset/" in f -# -# -# # Feature: Compound types correctly round-trip -# def test_compund(setup_teardown_file): -# """Compound types are read back in correct order.""" -# f = setup_teardown_file[3] -# grp = f.create_group("test") -# -# dt = np.dtype( [('weight', np.float64), -# ('cputime', np.float64), -# ('walltime', np.float64), -# ('parents_offset', np.uint32), -# ('n_parents', np.uint32), -# ('status', np.uint8), -# ('endpoint_type', np.uint8)]) -# -# testdata = np.ndarray((16,), dtype=dt) -# for key in dt.fields: -# testdata[key] = np.random.random((16,))*100 -# -# # print(testdata) -# -# grp['test'] = testdata -# outdata = grp['test'][()] -# assert np.all(outdata == testdata) -# assert outdata.dtype == testdata.dtype -# -# def test_assign(setup_teardown_file): -# f = setup_teardown_file[3] -# grp = f.create_group("test") -# -# dt = np.dtype([('weight', (np.float64, 3)), -# ('endpoint_type', np.uint8),]) -# -# testdata = np.ndarray((16,), dtype=dt) -# for key in dt.fields: -# testdata[key] = np.random.random(size=testdata[key].shape)*100 -# -# ds = grp.create_dataset('test', (16,), dtype=dt) -# for key in dt.fields: -# ds[key] = testdata[key] -# -# outdata = f['test']["test"][()] -# -# assert np.all(outdata == testdata) -# assert outdata.dtype == testdata.dtype -# -# -# -# -# def test_set_data(setup_teardown_file): -# """Set data works correctly.""" -# f = setup_teardown_file[3] -# grp = f.create_group("test") -# -# testdata = np.ones((10, 2)) -# grp['testdata'] = testdata -# outdata = grp['testdata'][()] -# assert np.all(outdata == testdata) -# assert outdata.dtype == testdata.dtype -# -# grp['testdata'] = testdata -# -# -# -# -# def test_eq_false(setup_teardown_file): -# """__eq__.""" -# f = setup_teardown_file[3] -# grp = f.create_group("test") -# -# dset = grp.create_dataset('foo', data=1) -# dset2 = grp.create_dataset('foobar', (2, 2)) -# -# assert dset != dset2 -# assert not dset == 2 -# -# def test_eq(setup_teardown_file): -# """__eq__.""" -# f = setup_teardown_file[3] -# grp = f.create_group("test") -# -# dset = grp.create_dataset('foo', data=np.ones((2, 2))) -# -# assert dset == dset -# -# -# def test_mmap(setup_teardown_file): -# """Test that changes to a mmap loaded numpy file is written to disk""" -# f = setup_teardown_file[3] -# grp = f.create_group("test") -# -# dset = grp.create_dataset('foo', (10**3, 10**3), fillvalue=2) -# dset[1, 1] = 100 -# -# tmp_file = np.load(str(setup_teardown_file[1] / "test" / "foo" / "data.npy")) -# -# assert dset.data[1, 1] == 100 -# assert tmp_file[1, 1] == 100 -# -# -# def test_modify_view(setup_teardown_file): -# f = setup_teardown_file[3] -# dataset = f.create_dataset("mydata", data=np.array([1, 2, 3, 4, 5, 6, 7, 8])) -# dataset[3:5] = np.array([8, 9]) -# assert np.array_equal(f["mydata"][3:5], np.array([8, 9])) -# view = dataset[3:5] -# view[0] = 10 -# assert f["mydata"][3] == 10 -# -# -# def test_single_index(setup_teardown_file): -# """Single-element selection with [index] yields array scalar.""" -# f = setup_teardown_file[3] -# dset = f.create_dataset('x', (1,), dtype='i1') -# out = dset[0] -# assert isinstance(out, np.int8) -# -# def test_single_null(setup_teardown_file): -# """Single-element selection with [()] yields ndarray.""" -# f = setup_teardown_file[3] -# -# dset = f.create_dataset('x', (1,), dtype='i1') -# out = dset[()] -# assert isinstance(out, np.ndarray) -# assert out.shape == (1,) -# -# def test_scalar_index(setup_teardown_file): -# """Slicing with [...] yields scalar ndarray.""" -# f = setup_teardown_file[3] -# -# dset = f.create_dataset('x', shape=(), dtype='f') -# out = dset[...] -# assert isinstance(out, np.ndarray) -# assert out.shape == () -# -# def test_scalar_null(setup_teardown_file): -# """Slicing with [()] yields array scalar.""" -# f = setup_teardown_file[3] -# -# dset = f.create_dataset('x', shape=(), dtype='i1') -# out = dset[()] -# -# assert out.dtype == "int8" -# -# def test_compound_index(setup_teardown_file): -# """Compound scalar is numpy.void, not tuple.""" -# f = setup_teardown_file[3] -# -# dt = np.dtype([('a', 'i4'), ('b', 'f8')]) -# v = np.ones((4,), dtype=dt) -# dset = f.create_dataset('foo', (4,), data=v) -# assert dset[0] == v[0] -# assert isinstance(dset[0], np.void) -# -# -# # Feature: Simple NumPy-style slices (start:stop:step) are supported. -# -# def test_negative_stop(setup_teardown_file): -# """Negative stop indexes work as they do in NumPy.""" -# f = setup_teardown_file[3] -# -# arr = np.arange(10) -# dset = f.create_dataset('x', data=arr) -# -# assert np.array_equal(dset[2:-2], arr[2:-2]) -# -# -# # Feature: Array types are handled appropriately -# -# def test_read(setup_teardown_file): -# """Read arrays tack array dimensions onto end of shape tuple.""" -# f = setup_teardown_file[3] -# -# dt = np.dtype('(3,)f8') -# dset = f.create_dataset('x', (10,), dtype=dt) -# # TODO implement this -# # assert dset.shape == (10,) -# # assert dset.dtype == dt -# -# # Full read -# out = dset[...] -# assert out.dtype == np.dtype('f8') -# assert out.shape == (10, 3) -# -# # Single element -# out = dset[0] -# assert out.dtype == np.dtype('f8') -# assert out.shape == (3,) -# -# # Range -# out = dset[2:8:2] -# assert out.dtype == np.dtype('f8') -# assert out.shape == (3, 3) -# -# def test_write_broadcast(setup_teardown_file): -# """Array fill from constant is supported.""" -# f = setup_teardown_file[3] -# -# dt = np.dtype('(3,)i') -# -# dset = f.create_dataset('x', (10,), dtype=dt) -# dset[...] = 42 -# -# -# -# def test_write_element(setup_teardown_file): -# """Write a single element to the array.""" -# f = setup_teardown_file[3] -# -# dt = np.dtype('(3,)f8') -# dset = f.create_dataset('x', (10,), dtype=dt) -# -# data = np.array([1, 2, 3.0]) -# dset[4] = data -# -# out = dset[4] -# assert np.all(out == data) -# -# -# def test_write_slices(setup_teardown_file): -# """Write slices to array type.""" -# f = setup_teardown_file[3] -# -# dt = np.dtype('(3,)i') -# -# data1 = np.ones((2, ), dtype=dt) -# data2 = np.ones((4, 5), dtype=dt) -# -# dset = f.create_dataset('x', (10, 9, 11), dtype=dt) -# -# dset[0, 0, 2:4] = data1 -# assert np.array_equal(dset[0, 0, 2:4], data1) -# -# dset[3, 1:5, 6:11] = data2 -# assert np.array_equal(dset[3, 1:5, 6:11], data2) -# -# -# def test_roundtrip(setup_teardown_file): -# """Read the contents of an array and write them back.""" -# f = setup_teardown_file[3] -# dt = np.dtype('(3,)f8') -# dset = f.create_dataset('x', (10,), dtype=dt) -# -# out = dset[...] -# dset[...] = out -# -# assert np.all(dset[...] == out) -# -# -# -# # Feature Slices resulting in empty arrays -# -# -# def test_slice_zero_length_dimension(setup_teardown_file): -# """Slice a dataset with a zero in its shape vector -# along the zero-length dimension.""" -# f = setup_teardown_file[3] -# -# for i, shape in enumerate([(0,), (0, 3), (0, 2, 1)]): -# dset = f.create_dataset('x%d'%i, shape, dtype=np.int) -# assert dset.shape == shape -# out = dset[...] -# assert isinstance(out, np.ndarray) -# assert out.shape == shape -# out = dset[:] -# assert isinstance(out, np.ndarray) -# assert out.shape == shape -# if len(shape) > 1: -# out = dset[:, :1] -# assert isinstance(out, np.ndarray) -# assert out.shape[:2] == (0, 1) -# -# def test_slice_other_dimension(setup_teardown_file): -# """Slice a dataset with a zero in its shape vector -# along a non-zero-length dimension.""" -# f = setup_teardown_file[3] -# -# for i, shape in enumerate([(3, 0), (1, 2, 0), (2, 0, 1)]): -# dset = f.create_dataset('x%d'%i, shape, dtype=np.int) -# assert dset.shape == shape -# out = dset[:1] -# assert isinstance(out, np.ndarray) -# assert out.shape == (1,)+shape[1:] -# -# def test_slice_of_length_zero(setup_teardown_file): -# """Get a slice of length zero from a non-empty dataset.""" -# f = setup_teardown_file[3] -# -# for i, shape in enumerate([(3, ), (2, 2, ), (2, 1, 5)]): -# dset = f.create_dataset('x%d'%i, data=np.zeros(shape, np.int)) -# assert dset.shape == shape -# out = dset[1:1] -# assert isinstance(out, np.ndarray) -# assert out.shape == (0,)+shape[1:] -# -# def test_modify_all(setup_teardown_file): -# f = setup_teardown_file[3] -# dset = f.create_dataset("test", data=np.arange(10)) -# dset.data = np.ones(4) -# assert np.all(dset.data == np.ones(4)) +# Feature: Size of first axis is available via Python's len +def test_len(setup_teardown_file): + """len().""" + f = setup_teardown_file[3] + grp = f.create_group("test") + data = pd.DataFrame(np.zeros((3, 10))) + dset = grp.require_dataset('bar', data=data) + assert len(dset) == 3 + +# Feature: Iterating over a dataset yields index keys + +def test_iter(setup_teardown_file): + """Iterating over a dataset yields rows.""" + f = setup_teardown_file[3] + grp = f.create_group("test") + + values = np.arange(30, dtype='f').reshape((10, 3)) + data = pd.DataFrame(values) + dset = grp.create_dataset('foo', data=data) + for x, y in zip(dset, data): + assert x == str(y) # NOTE feather converts int names to str + + +def test_set_data(setup_teardown_file): + """Set data works correctly.""" + f = setup_teardown_file[3] + grp = f.create_group("test") + + testdata = pd.DataFrame(np.ones((10, 2))) + grp['testdata'] = testdata + outdata = grp['testdata'].data + assert dataframe_equal(testdata, outdata) + + grp['testdata'] = testdata diff --git a/tests/test_dataset.py b/tests/test_dataset.py index 599d2ac..b16d3b8 100644 --- a/tests/test_dataset.py +++ b/tests/test_dataset.py @@ -171,6 +171,17 @@ def test_shape_conflict(setup_teardown_file): grp.require_dataset('foo', (10, 4), 'f') +def test_create_dtype_object(setup_teardown_file): + """Assignement of variable-length byte string produces a fixed-length + ascii dataset """ + f = setup_teardown_file[3] + grp = f.create_group("test") + # one needs object dtype in order to be able to change length of string with setitem + data = np.array(['aaaa', 'aaaaa'], dtype=object) + with pytest.raises(ValueError): + grp.create_dataset('foo', data=data) + + def test_type_confict(setup_teardown_file): """require_dataset with object type conflict yields TypeError.""" f = setup_teardown_file[3]