ahamilton/eris
1
0
Fork 0
Code Releases Activity

Load the summary asynchronously for quick startup.

Browse source 
- Interface starts before loading begins.
- Split out summary info into separate storage.
- Fixed iteration of PagedList.
- Not saving partially loaded summary during load.
This commit is contained in:
Andrew Hamilton 2020-04-09 11:08:39 +10:00
parent c9376e2bd3
commit 2704ccc9c3
5 changed files with 103 additions and 52 deletions
Download patch file Download diff file Expand all files Collapse all files

137
eris/__main__.py
View file

@ -22,6 +22,7 @@ import functools
import gzip
import importlib
import importlib.resources
import itertools
import math
import multiprocessing
import os
@ -43,6 +44,7 @@ from eris import terminal
from eris import termstr
from eris import tools
from eris import worker
from eris import paged_list
USAGE = """
@ -86,10 +88,12 @@ KEYS_DOC = """Keys:
class Entry:
def __init__(self, path, results, summary, highlighted=None,
MAX_WIDTH = 0
def __init__(self, path, results, change_time, highlighted=None,
set_results=True):
self.path = path
self.summary = summary
self.change_time = change_time
self.highlighted = highlighted
self.results = results
if set_results:
@ -111,8 +115,8 @@ class Entry:
def appearance_min(self):
if self.appearance_cache is None \
or self.last_width != self.summary._max_width:
self.last_width = self.summary._max_width
or self.last_width != Entry.MAX_WIDTH:
self.last_width = Entry.MAX_WIDTH
if self.highlighted is not None:
self.results[self.highlighted].is_highlighted = True
row_appearance = self.widget.appearance_min()
@ -131,7 +135,7 @@ class Entry:
html_parts.append(result_html)
styles.update(result_styles)
path = tools.path_colored(self.path)
padding = " " * (self.summary._max_width - len(self.widget) + 1)
padding = " " * (Entry.MAX_WIDTH - len(self.widget) + 1)
path_html, path_styles = termstr.TermStr(padding + path).as_html()
return "".join(html_parts) + path_html, styles.union(path_styles)
@ -206,22 +210,44 @@ class Summary:
self._root_path = root_path
self._jobs_added_event = jobs_added_event
self._view_widget = fill3.View.from_widget(self)
self.__cursor_position = (0, 0)
self.closest_placeholder_generator = None
self._cache = {}
self.is_directory_sort = True
self._max_width = 0
self._old_entries = []
self.reset()
def reset(self):
self.__cursor_position = (0, 0)
Entry.MAX_WIDTH = 0
self._max_path_length = 0
self._entries = sortedcontainers.SortedList([], key=directory_sort)
self.result_total = 0
self.completed_total = 0
self.is_loaded = False
self.closest_placeholder_generator = None
sort_func = directory_sort if self.is_directory_sort else type_sort
self._entries = sortedcontainers.SortedList([], key=sort_func)
def __getstate__(self):
state = self.__dict__.copy()
state["closest_placeholder_generator"] = None
state["_jobs_added_event"] = None
summary_path = os.path.join(tools.CACHE_PATH, "summary_dir")
open_compressed = functools.partial(gzip.open, compresslevel=1)
x, y = self.cursor_position()
if y == 0:
entries = []
else:
current_entry = self._entries[y]
del self._entries[y]
entries = itertools.chain([current_entry], self._entries)
state["_old_entries"] = paged_list.PagedList(
entries, summary_path, 2000, 1, exist_ok=True,
open_func=open_compressed)
state["_entries"] = None
return state
def __setstate__(self, state):
self.__dict__ = state
self.reset()
@property
def _cursor_position(self):
return self.__cursor_position
@ -238,24 +264,16 @@ class Summary:
self._entries, key=key_func)
self.closest_placeholder_generator = None
def file_added(self, path):
full_path = os.path.join(self._root_path, path)
try:
change_time = os.stat(full_path).st_ctime
except FileNotFoundError:
def add_entry(self, entry):
if entry in self._entries:
return
row = []
change_time = self._cache.setdefault(path, change_time)
for tool in tools.tools_for_path(path):
result = tools.Result(path, tool)
for result in entry:
self.result_total += 1
if result.is_completed:
self.completed_total += 1
row.append(result)
self._max_width = max(len(row), self._max_width)
self._max_path_length = max(len(path) - len("./"),
Entry.MAX_WIDTH = max(len(entry), Entry.MAX_WIDTH)
self._max_path_length = max(len(entry.path) - len("./"),
self._max_path_length)
entry = Entry(path, row, self)
self._entries.add(entry)
entry_index = self._entries.index(entry)
x, y = self._cursor_position
@ -264,10 +282,20 @@ class Summary:
self._jobs_added_event.set()
self.closest_placeholder_generator = None
def file_deleted(self, path, check=True):
if check and os.path.exists(os.path.join(self._root_path, path)):
def on_file_added(self, path):
full_path = os.path.join(self._root_path, path)
try:
change_time = os.stat(full_path).st_ctime
except FileNotFoundError:
return
entry = Entry(path, [], self)
row = [tools.Result(path, tool) for tool in tools.tools_for_path(path)]
entry = Entry(path, row, change_time)
self.add_entry(entry)
def on_file_deleted(self, path):
if os.path.exists(os.path.join(self._root_path, path)):
return
entry = Entry(path, [], None)
try:
index = self._entries.index(entry)
except ValueError:
@ -275,7 +303,6 @@ class Summary:
x, y = self._cursor_position
if index < y:
self.scroll(0, 1)
del self._cache[path]
for result in self._entries[index]:
if result.is_completed:
self.completed_total -= 1
@ -283,8 +310,8 @@ class Summary:
result.delete()
row = self._entries[index]
self._entries.pop(index)
if len(row) == self._max_width:
self._max_width = max((len(entry) for entry in self._entries),
if len(row) == Entry.MAX_WIDTH:
Entry.MAX_WIDTH = max((len(entry) for entry in self._entries),
default=0)
if (len(path) - 2) == self._max_path_length:
self._max_path_length = max(((len(entry.path) - 2)
@ -294,14 +321,15 @@ class Summary:
self._cursor_position = x, y - 1
self.closest_placeholder_generator = None
def file_modified(self, path):
entry = Entry(path, [], self)
def on_file_modified(self, path):
entry = Entry(path, [], None)
try:
entry_index = self._entries.index(entry)
except ValueError:
return
for result in self._entries[entry_index]:
self.refresh_result(result, only_completed=False)
self.closest_placeholder_generator = None
@contextlib.contextmanager
def keep_selection(self):
@ -320,25 +348,37 @@ class Summary:
self._cursor_position = (x, len(self._entries) - 1)
async def sync_with_filesystem(self, log=None):
log.log_message("Started syncing filesystem…")
start_time = time.time()
cache = {}
log.log_message("Started loading summary…")
for index, entry in enumerate(self._old_entries):
if index != 0 and index % 5000 == 0:
log.log_message(f"Loaded {index} files…")
await asyncio.sleep(0)
self.add_entry(entry)
cache[entry.path] = entry.change_time
duration = time.time() - start_time
log.log_message(f"Finished loading summary. {round(duration, 2)} secs")
self.is_loaded = True
log.log_message("Started sync with filesystem…")
start_time = time.time()
all_paths = set()
for path in fix_paths(self._root_path, codebase_files(self._root_path)):
await asyncio.sleep(0)
all_paths.add(path)
if path in self._cache:
if path in cache:
full_path = os.path.join(self._root_path, path)
change_time = os.stat(full_path).st_ctime
if change_time != self._cache[path]:
self._cache[path] = change_time
self.file_modified(path)
if change_time != cache[path]:
cache[path] = change_time
self.on_file_modified(path)
else:
self.file_added(path)
for path in self._cache.keys() - all_paths:
self.on_file_added(path)
for path in cache.keys() - all_paths:
await asyncio.sleep(0)
self.file_deleted(path)
self.on_file_deleted(path)
duration = time.time() - start_time
log.log_message(f"Finished syncing filesystem. {round(duration, 2)} secs")
log.log_message(f"Finished sync with filesystem. {round(duration, 2)} secs")
def _sweep_up(self, x, y):
yield from reversed(self._entries[y][:x])
@ -375,7 +415,7 @@ class Summary:
return await self.closest_placeholder_generator.asend(None)
def appearance_dimensions(self):
return self._max_path_length + 1 + self._max_width, len(self._entries)
return self._max_path_length + 1 + Entry.MAX_WIDTH, len(self._entries)
def appearance_interval(self, interval):
start_y, end_y = interval
@ -1072,12 +1112,12 @@ def on_filesystem_event(event, summary, root_path, appearance_changed_event):
path = list(fix_paths(root_path, [event.pathname]))[0]
if is_path_excluded(path[2:]):
return
inotify_actions = {pyinotify.IN_CREATE: summary.file_added,
pyinotify.IN_MOVED_TO: summary.file_added,
pyinotify.IN_DELETE: summary.file_deleted,
pyinotify.IN_MOVED_FROM: summary.file_deleted,
pyinotify.IN_ATTRIB: summary.file_modified,
pyinotify.IN_CLOSE_WRITE: summary.file_modified}
inotify_actions = {pyinotify.IN_CREATE: summary.on_file_added,
pyinotify.IN_MOVED_TO: summary.on_file_added,
pyinotify.IN_DELETE: summary.on_file_deleted,
pyinotify.IN_MOVED_FROM: summary.on_file_deleted,
pyinotify.IN_ATTRIB: summary.on_file_modified,
pyinotify.IN_CLOSE_WRITE: summary.on_file_modified}
if event.mask not in inotify_actions:
return
try:
@ -1124,7 +1164,8 @@ def main(root_path, loop, worker_count=None, editor_command=None, theme=None,
log.log_message("Program stopped.")
finally:
notifier.stop()
screen.save()
if summary.is_loaded:
screen.save()
@contextlib.contextmanager

8
eris/paged_list.py
View file

@ -29,6 +29,7 @@ class PagedList:
shutil.rmtree(tmp_dir, ignore_errors=True)
shutil.rmtree(pages_dir, ignore_errors=True)
os.makedirs(tmp_dir)
index = 0
for index, page in enumerate(batch(list_, page_size)):
pickle_path = os.path.join(tmp_dir, str(index))
with self.open_func(pickle_path, "wb") as file_:
@ -43,8 +44,11 @@ class PagedList:
def _get_page_org(self, index): # This is cached, see setup_page_cache.
pickle_path = os.path.join(self.pages_dir, str(index))
with self.open_func(pickle_path, "rb") as file_:
return pickle.load(file_)
try:
with self.open_func(pickle_path, "rb") as file_:
return pickle.load(file_)
except FileNotFoundError:
raise IndexError
def __getitem__(self, index):
if isinstance(index, slice):

2
eris/worker.py
View file

@ -55,7 +55,7 @@ class Worker:
await self.result.run(log, appearance_changed_event, self)
self.result.compression = self.compression
Worker.unsaved_jobs_total += 1
if Worker.unsaved_jobs_total == 2000:
if Worker.unsaved_jobs_total == 5000 and summary.is_loaded:
log.log_message(Worker.AUTOSAVE_MESSAGE)
screen.save()
summary.completed_total += 1

2
tests/__main___test.py
View file

@ -159,7 +159,7 @@ class SummarySyncWithFilesystemTestCase(unittest.TestCase):
self.assertEqual(self.summary.completed_total, completed_total)
self.assertEqual(self.summary.result_total, result_total)
max_width = max((len(row) for row in self.summary._entries), default=0)
self.assertEqual(self.summary._max_width, max_width)
self.assertEqual(__main__.Entry.MAX_WIDTH, max_width)
max_path_length = max(
(len(row.path) - 2 for row in self.summary._entries), default=0)
self.assertEqual(self.summary._max_path_length, max_path_length)

6
tests/paged_list_test.py
View file

@ -31,6 +31,12 @@ class PagedListTestCase(unittest.TestCase):
self.assertEqual(list_[1:5], [4, 5, 6, 7])
self.assertEqual(list_[:2], [3, 4])
self.assertEqual(list_[2:], [5, 6, 7, 8])
self.assertEqual(list(list_), [3, 4, 5, 6, 7, 8])
self.assertRaises(IndexError, list_.__getitem__, 6)
with tempfile.TemporaryDirectory() as temp_dir:
list_ = paged_list.PagedList([], temp_dir, 2, 2)
self.assertRaises(IndexError, list_.__getitem__, 0)
# self.assertEqual(list_[3:4], []) FIX
def test_pickling(self):
with tempfile.TemporaryDirectory() as temp_dir: