From 81960d68a98e2caa71bb052de45b7b927818943a Mon Sep 17 00:00:00 2001 From: Dirk Alders Date: Sun, 13 Oct 2024 09:34:09 +0200 Subject: [PATCH] Addad system pages capability and improved meta diff view --- data/{pages => system-pages}/index/meta.json | 0 data/{pages => system-pages}/index/page | 0 data/system-pages/startpage/meta.json | 5 + data/system-pages/startpage/page | 5 + pages/page.py | 358 ++++++++++++------- pages/search.py | 34 +- pages/views.py | 12 +- piki/settings.py | 1 + themes | 2 +- 9 files changed, 268 insertions(+), 149 deletions(-) rename data/{pages => system-pages}/index/meta.json (100%) rename data/{pages => system-pages}/index/page (100%) create mode 100644 data/system-pages/startpage/meta.json create mode 100644 data/system-pages/startpage/page diff --git a/data/pages/index/meta.json b/data/system-pages/index/meta.json similarity index 100% rename from data/pages/index/meta.json rename to data/system-pages/index/meta.json diff --git a/data/pages/index/page b/data/system-pages/index/page similarity index 100% rename from data/pages/index/page rename to data/system-pages/index/page diff --git a/data/system-pages/startpage/meta.json b/data/system-pages/startpage/meta.json new file mode 100644 index 0000000..b5794c1 --- /dev/null +++ b/data/system-pages/startpage/meta.json @@ -0,0 +1,5 @@ +{ + "creation_time": 1728465495, + "modified_time": 1728649989, + "modified_user": "system-page" +} diff --git a/data/system-pages/startpage/page b/data/system-pages/startpage/page new file mode 100644 index 0000000..1d9b218 --- /dev/null +++ b/data/system-pages/startpage/page @@ -0,0 +1,5 @@ += Default startpage +Edit this page to get your own first startpage. + +If you need need assistance to edit a page, visit the [[/helpview/main|help pages]]. + diff --git a/pages/page.py b/pages/page.py index e373c1b..3473514 100644 --- a/pages/page.py +++ b/pages/page.py @@ -1,3 +1,4 @@ +import difflib from django.conf import settings from django.utils.translation import gettext as _ import fstools @@ -13,36 +14,52 @@ from . import timestamp_to_datetime logger = logging.getLogger(settings.ROOT_LOGGER_NAME).getChild(__name__) +SPLITCHAR = ":" +HISTORY_FOLDER_NAME = 'history' + + +def full_path_all_pages(expression="*"): + system_pages = fstools.dirlist(settings.SYSTEM_PAGES_ROOT, expression=expression, rekursive=False) + system_pages = [os.path.join(settings.PAGES_ROOT, os.path.basename(path)) for path in system_pages] + pages = fstools.dirlist(settings.PAGES_ROOT, expression=expression, rekursive=False) + return list(set(system_pages + pages)) + + class meta_data(dict): + META_FILE_NAME = 'meta.json' + # KEY_CREATION_TIME = "creation_time" KEY_MODIFIED_TIME = "modified_time" KEY_MODIFIED_USER = "modified_user" KEY_TAGS = "tags" - def __init__(self, meta_filename, page_exists): - self._meta_filename = meta_filename - + def __init__(self, path, history_version=None): + self._path = path + self._history_version = history_version + # # Load data from disk try: - with open(meta_filename, 'r') as fh: + with open(self.filename, 'r') as fh: super().__init__(json.load(fh)) except (FileNotFoundError, json.decoder.JSONDecodeError) as e: super().__init__() - # Add missing information to meta_data - missing_keys = False - if self.KEY_CREATION_TIME not in self: - missing_keys = True - self[self.KEY_CREATION_TIME] = int(time.time()) - if self.KEY_MODIFIED_TIME not in self: - self[self.KEY_MODIFIED_TIME] = self[self.KEY_CREATION_TIME] - if missing_keys and page_exists: - self.save() + @property + def filename(self): + if not self._history_version: + return os.path.join(self._path, self.META_FILE_NAME) + else: + return self.history_filename(self._history_version) + + def history_filename(self, history_version): + return os.path.join(self._path, HISTORY_FOLDER_NAME, "%05d_%s" % (history_version, self.META_FILE_NAME)) def update(self, username, tags): if username: self[self.KEY_MODIFIED_TIME] = int(time.time()) self[self.KEY_MODIFIED_USER] = username + if self.KEY_CREATION_TIME not in self: + self[self.KEY_CREATION_TIME] = self[self.KEY_MODIFIED_TIME] if tags: self[self.KEY_TAGS] = tags # @@ -50,30 +67,27 @@ class meta_data(dict): self.save() def save(self): - with open(self._meta_filename, 'w') as fh: - json.dump(self, fh, indent=4) + if self._history_version: + logger.error("A history version %05d can not be updated!", self._history_version) + return False + else: + with open(self.filename, 'w') as fh: + json.dump(self, fh, indent=4) + return True + + def store_to_history(self, history_number): + history_filename = self.history_filename(history_number) + fstools.mkdir(os.path.dirname(history_filename)) + shutil.copy(self.filename, history_filename) -class base_page(object): +class page_data(object): PAGE_FILE_NAME = 'page' - META_FILE_NAME = 'meta.json' - HISTORY_FOLDER_NAME = 'history' - SPLITCHAR = ":" def __init__(self, path, history_version=None): self._history_version = history_version - # - if path.startswith(settings.PAGES_ROOT): - self._path = path - else: - self._path = os.path.join(settings.PAGES_ROOT, path.replace("/", 2*self.SPLITCHAR)) + self._path = path self._raw_page_src = None - # - self._meta_data = meta_data(self._meta_filename, self.is_available()) - - @property - def modified_time(self): - return self._meta_data.get(self._meta_data.KEY_MODIFIED_TIME) def _load_page_src(self): if self._raw_page_src is None: @@ -83,43 +97,20 @@ class base_page(object): except FileNotFoundError: self._raw_page_src = "" - def history_numbers_list(self): - history_folder = os.path.join(self._path, self.HISTORY_FOLDER_NAME) - fstools.mkdir(history_folder) - # identify last_history number - return list(set([int(os.path.basename(filename)[:5]) for filename in fstools.filelist(history_folder)])) + def update_required(self, page_txt): + return page_txt.replace("\r\n", "\n") != self.raw_page_src - def _store_history(self): - try: - hist_number = max(self.history_numbers_list()) + 1 - except ValueError: - hist_number = 1 # no history yet - # copy file to history folder - shutil.copy(self.filename, self.history_filename(hist_number)) - shutil.copy(self._meta_filename, self._history_meta_filename(hist_number)) - - def update_page(self, page_txt, tags): + def update_page(self, page_txt): if self._history_version: logger.error("A history version %05d can not be updated!", self._history_version) return False else: - from .search import update_item - if page_txt.replace("\r\n", "\n") != self.raw_page_src: - # Store page history - if self.raw_page_src: - self._store_history() - # save the new page content - fstools.mkdir(os.path.dirname(self.filename)) - with open(self.filename, 'w') as fh: - fh.write(page_txt) - # update metadata - page_changed = True - else: - page_changed = False - self._update_metadata(tags) - # update search index - update_item(self) - return page_changed + # save the new page content + fstools.mkdir(os.path.dirname(self.filename)) + with open(self.filename, 'w') as fh: + fh.write(page_txt) + self._raw_page_src = page_txt + return True @property def filename(self): @@ -129,24 +120,11 @@ class base_page(object): return self.history_filename(self._history_version) def history_filename(self, history_version): - return os.path.join(self._path, self.HISTORY_FOLDER_NAME, "%05d_%s" % (history_version, self.PAGE_FILE_NAME)) - - @property - def _meta_filename(self): - if not self._history_version: - return os.path.join(self._path, self.META_FILE_NAME) - else: - return self._history_meta_filename(self._history_version) - - def _history_meta_filename(self, history_version): - return os.path.join(self._path, self.HISTORY_FOLDER_NAME, "%05d_%s" % (history_version, self.META_FILE_NAME)) + return os.path.join(self._path, HISTORY_FOLDER_NAME, "%05d_%s" % (history_version, self.PAGE_FILE_NAME)) @property def rel_path(self): - return os.path.basename(self._path).replace(2*self.SPLITCHAR, "/") - - def rel_path_is_valid(self): - return not self.SPLITCHAR in self.rel_path + return os.path.basename(self._path).replace(2*SPLITCHAR, "/") def is_available(self): is_a = os.path.isfile(self.filename) @@ -163,23 +141,13 @@ class base_page(object): self._load_page_src() return self._raw_page_src - def _update_metadata(self, tags): - username = None - try: - if self._request.user.is_authenticated: - username = self._request.user.username - else: - logger.warning("Page edit without having a logged in user. This is not recommended. Check your access definitions!") - except AttributeError: - logger.exception("Page edit without having a request object. Check programming!") - self._meta_data.update(username, tags) - - @property - def page_tags(self): - return self._meta_data.get(self._meta_data.KEY_TAGS) + def store_to_history(self, history_number): + history_filename = self.history_filename(history_number) + fstools.mkdir(os.path.dirname(history_filename)) + shutil.copy(self.filename, history_filename) -class creole_page(base_page): +class page_django(page_data): FOLDER_ATTACHMENTS = "attachments" def __init__(self, request, path, history_version=None) -> None: @@ -197,18 +165,22 @@ class creole_page(base_page): messages.unavailable_msg_page(self._request, self.rel_path) return "" - def render_meta(self): - ctime = timestamp_to_datetime(self._request, self._meta_data.get(self._meta_data.KEY_CREATION_TIME)).strftime('%Y-%m-%d %H:%M') - mtime = timestamp_to_datetime(self._request, self._meta_data.get(self._meta_data.KEY_MODIFIED_TIME)).strftime('%Y-%m-%d %H:%M') - user = self._meta_data.get(self._meta_data.KEY_MODIFIED_USER) - tags = self._meta_data.get(self._meta_data.KEY_TAGS, "-") + def history_numbers_list(self): + history_folder = os.path.join(self._path, HISTORY_FOLDER_NAME) + return list(set([int(os.path.basename(filename)[:5]) for filename in fstools.filelist(history_folder)])) + + def render_meta(self, ctime, mtime, user, tags): + # + # Page meta data # meta = f'=== {_("Meta data")}\n' - meta += f'|{_("Created")}:|{ctime}|\n' - meta += f'|{_("Modified")}:|{mtime}|\n' + meta += f'|{_("Created")}:|{timestamp_to_datetime(self._request, ctime)}|\n' + meta += f'|{_("Modified")}:|{timestamp_to_datetime(self._request, mtime)}|\n' meta += f'|{_("Editor")}|{user}|\n' meta += f'|{_("Tags")}|{tags}|\n' # + # List of hostory page versions + # hnl = self.history_numbers_list() if hnl: meta += f'=== {_("History")}\n' @@ -216,40 +188,27 @@ class creole_page(base_page): # Current name = _("Current") meta += f"| {name} \ - | {timestamp_to_datetime(self._request, self.modified_time)} \ + | {timestamp_to_datetime(self._request, mtime)} \ | [[{url_page(self._request, self.rel_path)} | Page]] \ | [[{url_page(self._request, self.rel_path, meta=None)} | Meta]]\n" # History for num in reversed(hnl): - p = creole_page(self._request, self._path, history_version=num) + p = page_wrapped(self._request, self._path, history_version=num) meta += f"| {num} \ | {timestamp_to_datetime(self._request, p.modified_time)} \ | [[{url_page(self._request, p.rel_path, history=num)} | Page]] \ - | [[{url_page(self._request, p.rel_path, meta=None, history=num)} | Meta]]\n" - # - meta += f'=== {_("Page content")}\n' - if not self._history_version: - meta += '{{{\n%s\n}}}\n' % self.raw_page_src - else: - c = creole_page(self._request, self.rel_path) - meta += "| =Current | =This |\n" + | [[{url_page(self._request, p.rel_path, meta=None, history=num)} | Meta]] (with page changes)\n" + # Diff + html_diff = "" + if self._history_version: + meta += f'=== {_("Page differences")}\n' + # + c = page_django(self._request, self._path) left_lines = c.raw_page_src.splitlines() right_lines = self.raw_page_src.splitlines() - while len(left_lines) + len(right_lines) > 0: - try: - left = left_lines.pop(0) - except IndexError: - left = "" - try: - right = right_lines.pop(0) - except IndexError: - right = "" - if left == right: - meta += "| {{{ %s }}} | {{{ %s }}} |\n" % (left, right) - else: - meta += "| **{{{ %s }}}** | **{{{ %s }}}** |\n" % (left, right) + html_diff = difflib.HtmlDiff(wrapcolumn=80).make_table(left_lines, right_lines) # - return mycreole.render_simple(meta) + return mycreole.render_simple(meta) + html_diff def render_text(self, request, txt): macros = { @@ -289,11 +248,12 @@ class creole_page(base_page): expression = "*" parent_rel_path = "" else: - expression = os.path.basename(self._path) + 2 * self.SPLITCHAR + "*" + expression = os.path.basename(self._path) + 2 * SPLITCHAR + "*" parent_rel_path = self.rel_path + # pl = page_list( self._request, - [creole_page(self._request, path) for path in fstools.dirlist(settings.PAGES_ROOT, expression=expression, rekursive=False)] + [page_django(self._request, path) for path in full_path_all_pages(expression)] ) return pl.html_list(depth=depth, filter_str=filter_str, parent_rel_path=parent_rel_path) @@ -327,3 +287,151 @@ class page_list(list): def html_list(self, depth=9999, filter_str='', parent_rel_path=''): return mycreole.render_simple(self.creole_list(depth, filter_str, parent_rel_path)) + + +class page_wrapped(object): + """ + This class holds different page and meta instances and decides which will be used in which case. + """ + + def __init__(self, request, path, history_version=None): + """_summary_ + + Args: + request (_type_): The django request or None (if None, the page functionality is limited) + path (_type_): A rel_path of the django page or the filesystem path to the page + history_version (_type_, optional): The history version of the page to be created + """ + self._request = request + # + page_path = self.__page_path__(path) + system_page_path = self.__system_page_path__(path) + # Page + if request: + self._page = page_django(request, page_path, history_version=history_version) + else: + self._page = page_data(page_path, history_version=history_version) + self._page_meta = meta_data(page_path, history_version=history_version) + # System page + if request: + self._system_page = page_django(request, system_page_path) + else: + self._system_page = page_data(system_page_path) + self._system_meta_data = meta_data(system_page_path) + + def __page_path__(self, path): + if path.startswith(settings.PAGES_ROOT): + # must be a filesystem path + return path + else: + # must be a relative url + return os.path.join(settings.PAGES_ROOT, path.replace("/", 2*SPLITCHAR)) + + def __system_page_path__(self, path): + return os.path.join(settings.SYSTEM_PAGES_ROOT, os.path.basename(path)) + + def __page_choose__(self): + if not self._page.is_available(): + return self._system_page + else: + return self._page + + def __meta_choose__(self): + if not self._page.is_available(): + return self._system_meta_data + else: + return self._page_meta + + def __store_history__(self): + if self._page.is_available(): + try: + history_number = max(self._page.history_numbers_list()) + 1 + except ValueError: + history_number = 1 # no history yet + self._page.store_to_history(history_number) + self._page_meta.store_to_history(history_number) + + # + # meta_data + # + @property + def creation_time(self): + meta = self.__meta_choose__() + rv = meta.get(meta.KEY_CREATION_TIME) + return rv + + @property + def modified_time(self): + meta = self.__meta_choose__() + rv = meta.get(meta.KEY_MODIFIED_TIME) + return rv + + @property + def modified_user(self): + meta = self.__meta_choose__() + rv = meta.get(meta.KEY_MODIFIED_USER) + return rv + + @property + def tags(self): + meta = self.__meta_choose__() + rv = meta.get(meta.KEY_TAGS) + return rv + + # + # page + # + @property + def attachment_path(self): + page = self.__page_choose__() + rv = page.attachment_path + return rv + + @property + def raw_page_src(self): + page = self.__page_choose__() + rv = page.raw_page_src + return rv + + @property + def rel_path(self): + page = self.__page_choose__() + rv = page.rel_path + return rv + + def render_meta(self): + page = self.__page_choose__() + rv = page.render_meta(self.creation_time, self.modified_time, self.modified_user, self.tags) + return rv + + def render_to_html(self): + page = self.__page_choose__() + rv = page.render_to_html() + return rv + + @property + def title(self): + page = self.__page_choose__() + rv = page.title + return rv + + def update_page(self, txt, tags): + if self._page.update_required(txt): + # Store history + self.__store_history__() + # Update page + rv = self._page.update_page(txt) + # Update meta data + username = None + try: + if self._request.user.is_authenticated: + username = self._request.user.username + else: + logger.warning("Page edit without having a logged in user. This is not recommended. Check your access definitions!") + except AttributeError: + logger.exception("Page edit without having a request object. Check programming!") + self._page_meta.update(username, tags) + # Update search index + from pages.search import update_item + update_item(self) + return rv diff --git a/pages/search.py b/pages/search.py index 04a4a38..8183d13 100644 --- a/pages/search.py +++ b/pages/search.py @@ -8,7 +8,7 @@ from whoosh.fields import Schema, ID, TEXT, DATETIME from whoosh.qparser.dateparse import DateParserPlugin from whoosh import index, qparser -from pages.page import base_page +from pages.page import page_wrapped logger = logging.getLogger(settings.ROOT_LOGGER_NAME).getChild(__name__) @@ -40,8 +40,8 @@ def create_index(): def rebuild_index(ix): page_path = fstools.dirlist(settings.PAGES_ROOT, rekursive=False) for path in page_path: - bp = base_page(path) - add_item(ix, bp) + pw = page_wrapped(None, path) + add_item(ix, pw) return len(page_path) @@ -56,19 +56,19 @@ def load_index(): return ix -def add_item(ix, bp: base_page): +def add_item(ix, pw: page_wrapped): # Define Standard data # data = dict( - id=bp.rel_path, + id=pw.rel_path, # - title=bp.title, - page_src=bp.raw_page_src, - tag=bp.page_tags, + title=pw.title, + page_src=pw.raw_page_src, + tag=pw.tags, # - creation_time=datetime.fromtimestamp(bp._meta_data.get(bp._meta_data.KEY_CREATION_TIME)), - modified_time=datetime.fromtimestamp(bp._meta_data.get(bp._meta_data.KEY_MODIFIED_TIME)), - modified_user=bp._meta_data.get(bp._meta_data.KEY_MODIFIED_USER) + creation_time=datetime.fromtimestamp(pw.creation_time), + modified_time=datetime.fromtimestamp(pw.modified_time), + modified_user=pw.modified_user ) with ix.writer() as w: logger.info('Adding document with id=%s to the search index.', data.get('id')) @@ -95,13 +95,13 @@ def whoosh_search(search_txt): return rpl -def delete_item(ix, bp: base_page): +def delete_item(ix, pw: page_wrapped): with ix.writer() as w: - logger.info('Removing document with id=%s from the search index.', bp.rel_path) - w.delete_by_term("task_id", bp.rel_path) + logger.info('Removing document with id=%s from the search index.', pw.rel_path) + w.delete_by_term("task_id", pw.rel_path) -def update_item(bp: base_page): +def update_item(pw: page_wrapped): ix = load_index() - delete_item(ix, bp) - add_item(ix, bp) + delete_item(ix, pw) + add_item(ix, pw) diff --git a/pages/views.py b/pages/views.py index 3161473..7186c99 100644 --- a/pages/views.py +++ b/pages/views.py @@ -15,7 +15,7 @@ from .context import context_adaption from .forms import EditForm from .help import help_pages import mycreole -from .page import creole_page, page_list +from .page import page_wrapped, page_list from .search import whoosh_search from themes import Context @@ -34,7 +34,7 @@ def page(request, rel_path): if history: history = int(history) # - p = creole_page(request, rel_path, history_version=history) + p = page_wrapped(request, rel_path, history_version=history) if access.read_page(request, rel_path): if meta: page_content = p.render_meta() @@ -66,9 +66,9 @@ def edit(request, rel_path): if history: history = int(history) # - p = creole_page(request, rel_path, history_version=history) + p = page_wrapped(request, rel_path, history_version=history) # - form = EditForm(page_data=p.raw_page_src, page_tags=p.page_tags) + form = EditForm(page_data=p.raw_page_src, page_tags=p.tags) # context_adaption( context, @@ -80,7 +80,7 @@ def edit(request, rel_path): ) return render(request, 'pages/page_form.html', context=context) else: - p = creole_page(request, rel_path) + p = page_wrapped(request, rel_path) # save = request.POST.get("save") page_txt = request.POST.get("page_txt") @@ -122,7 +122,7 @@ def search(request): if sr is None: django_messages.error(request, _('Invalid search pattern: %s') % repr(search_txt)) sr = [] - pl = page_list(request, [creole_page(request, rel_path) for rel_path in set(sr)]) + pl = page_list(request, [page_wrapped(request, rel_path) for rel_path in set(sr)]) # context_adaption( context, diff --git a/piki/settings.py b/piki/settings.py index 445b8a4..0203888 100644 --- a/piki/settings.py +++ b/piki/settings.py @@ -134,6 +134,7 @@ MYCREOLE_BAR = { 'menubar': 'pages.context.menubar', } +SYSTEM_PAGES_ROOT = os.path.join(BASE_DIR, 'data', 'system-pages') PAGES_ROOT = os.path.join(BASE_DIR, 'data', 'pages') WHOOSH_PATH = os.path.join(BASE_DIR, 'data', 'whoosh') diff --git a/themes b/themes index 83b0297..fe05827 160000 --- a/themes +++ b/themes @@ -1 +1 @@ -Subproject commit 83b0297d433101c64369994d7b7958ce84524cb9 +Subproject commit fe05827340092db16eda3159cb19da8bf7f5b093