Edit: test_buf.py
from .lib import TestBase, FileCreator from smmap.mman import ( SlidingWindowMapManager, StaticWindowMapManager ) from smmap.buf import SlidingWindowMapBuffer from random import randint from time import time import sys import os man_optimal = SlidingWindowMapManager() man_worst_case = SlidingWindowMapManager( window_size=TestBase.k_window_test_size // 100, max_memory_size=TestBase.k_window_test_size // 3, max_open_handles=15) static_man = StaticWindowMapManager() class TestBuf(TestBase): def test_basics(self): with FileCreator(self.k_window_test_size, "buffer_test") as fc: # invalid paths fail upon construction c = man_optimal.make_cursor(fc.path) self.assertRaises(ValueError, SlidingWindowMapBuffer, type(c)()) # invalid cursor self.assertRaises(ValueError, SlidingWindowMapBuffer, c, fc.size) # offset too large buf = SlidingWindowMapBuffer() # can create uninitailized buffers assert buf.cursor() is None # can call end access any time buf.end_access() buf.end_access() assert len(buf) == 0 # begin access can revive it, if the offset is suitable offset = 100 assert buf.begin_access(c, fc.size) == False assert buf.begin_access(c, offset) == True assert len(buf) == fc.size - offset assert buf.cursor().is_valid() # empty begin access keeps it valid on the same path, but alters the offset assert buf.begin_access() == True assert len(buf) == fc.size assert buf.cursor().is_valid() # simple access with open(fc.path, 'rb') as fp: data = fp.read() assert data[offset] == buf[0] assert data[offset:offset * 2] == buf[0:offset] # negative indices, partial slices assert buf[-1] == buf[len(buf) - 1] assert buf[-10:] == buf[len(buf) - 10:len(buf)] # end access makes its cursor invalid buf.end_access() assert not buf.cursor().is_valid() assert buf.cursor().is_associated() # but it remains associated # an empty begin access fixes it up again assert buf.begin_access() == True and buf.cursor().is_valid() del(buf) # ends access automatically del(c) assert man_optimal.num_file_handles() == 1 # PERFORMANCE # blast away with random access and a full mapping - we don't want to # exaggerate the manager's overhead, but measure the buffer overhead # We do it once with an optimal setting, and with a worse manager which # will produce small mappings only ! max_num_accesses = 100 fd = os.open(fc.path, os.O_RDONLY) for item in (fc.path, fd): for manager, man_id in ((man_optimal, 'optimal'), (man_worst_case, 'worst case'), (static_man, 'static optimal')): buf = SlidingWindowMapBuffer(manager.make_cursor(item)) assert manager.num_file_handles() == 1 for access_mode in range(2): # single, multi num_accesses_left = max_num_accesses num_bytes = 0 fsize = fc.size st = time() buf.begin_access() while num_accesses_left: num_accesses_left -= 1 if access_mode: # multi ofs_start = randint(0, fsize) ofs_end = randint(ofs_start, fsize) d = buf[ofs_start:ofs_end] assert len(d) == ofs_end - ofs_start assert d == data[ofs_start:ofs_end] num_bytes += len(d) del d else: pos = randint(0, fsize) assert buf[pos] == data[pos] num_bytes += 1 # END handle mode # END handle num accesses buf.end_access() assert manager.num_file_handles() assert manager.collect() assert manager.num_file_handles() == 0 elapsed = max(time() - st, 0.001) # prevent zero division errors on windows mb = float(1000 * 1000) mode_str = (access_mode and "slice") or "single byte" print("%s: Made %i random %s accesses to buffer created from %s reading a total of %f mb in %f s (%f mb/s)" % (man_id, max_num_accesses, mode_str, type(item), num_bytes / mb, elapsed, (num_bytes / mb) / elapsed), file=sys.stderr) # END handle access mode del buf # END for each manager # END for each input os.close(fd)
Cancel