Use mdutils, replace checkboxes, add link annotations (#5)

* Add dependency for MDUtils

* Add check for timestamps for deleted, trashed > 18000, as "no value" seems to be 18000 based on my notes

* Use MDUtils for Markdown generation, fix filenames with no title or trailing spaces

* Replace unicode checkbox symbols with markdown checkbox lists

* Add annotation links to the end of the note

* Rough first pass at downloading and adding image references to Markdown

keep_exporter/export.py

@@ -1,11 +1,13 @@
 #!/usr/bin/env python3
 
 import os, pathlib
+from typing import List
 import gkeepapi
 import frontmatter
 import click
 from pathvalidate import sanitize_filename
-
+from mdutils.mdutils import MdUtils
+from tempfile import NamedTemporaryFile
 
 def login(user_email: str, password: str) -> gkeepapi.Keep:
     keep = gkeepapi.Keep()
@@ -16,8 +18,33 @@ def login(user_email: str, password: str) -> gkeepapi.Keep:
 
     return keep
 
+def download_images(keep: gkeepapi.Keep, note_count: int, note: gkeepapi._node.Note, outpath) -> List[pathlib.Path]:
+    if not note.images: return []
+
+    ret = []
+
+    for image in note.images:
+        meta = image.blob.save()        
+        mimetype = meta['mimetype']
+        ocr = meta['extracted_text']
+
+        url = keep._media_api.get(image)
+        print("Downloading %s" % url)
+
+        imgfile = (
+            outpath
+            / f'{sanitize_filename(f"{note_count:04} - " + image.server_id,max_len=135)}.jpg'
+        )
+        
+        imgresp = keep._media_api._session.get(url)
+        with open(imgfile, 'wb') as f:
+            f.write( imgresp.content )
+
+        ret.append( imgfile )
+
+    return ret
 
-def process_note(note: gkeepapi._node.Note) -> frontmatter.Post:
+def build_frontmatter(note: gkeepapi._node.Note, markdown: str) -> frontmatter.Post:
     metadata = {
         "id": note.id,
         "title": note.title,
@@ -36,13 +63,55 @@ def process_note(note: gkeepapi._node.Note) -> frontmatter.Post:
         },
     }
 
-    if note.timestamps.trashed:
+    # gkeepapi appears to be treating "0" as a timestamp rather than null
+    if note.timestamps.trashed and note.timestamps.trashed.year > 1970:
         metadata["timestamps"]["trashed"] = note.timestamps.trashed.timestamp()
-    if note.timestamps.deleted:
+    if note.timestamps.deleted and note.timestamps.deleted.year > 1970:
         metadata["timestamps"]["deleted"] = note.timestamps.deleted.timestamp()
 
-    return frontmatter.Post(note.text, handler=None, **metadata)
-
+    return frontmatter.Post(markdown, handler=None, **metadata)
+
+def build_markdown(note: gkeepapi._node.Note, images: List[pathlib.Path]) -> str:
+    # mdutils demands a filename to write to
+    # and doesn't seem to support working in memory
+    with NamedTemporaryFile() as f:
+        doc = MdUtils(file_name=f.name)
+
+        doc.new_header(1, note.title)
+        doc.new_header(2, "Note")
+
+        text = note.text
+        text = text.replace('☑ ', '- [X] ')
+        text = text.replace('☐ ', '- [ ] ')
+
+        doc.new_paragraph(text)
+
+        if note.annotations.links:
+            doc.new_line()
+            doc.new_line()
+            doc.new_header(2, "Links")
+            doc.new_list(
+                [
+                    doc.new_inline_link(link=link.url, text=link.title)
+                    for link in note.annotations.links
+                ]
+            )
+
+        if images:
+            doc.new_header(2, "Attached Images")
+            doc.new_line()
+            doc.new_line()
+
+            for image in images:
+                doc.new_line( doc.new_inline_image("", image.name) )
+
+        # doc.create_md_file()
+        # create_md_file writes out:
+        #    data=self.title + self.table_of_contents + self.file_data_text + self.reference.get_references_as_markdown()
+        # but we only care about file_data_text
+        # since we're not generating a TOC or references
+        # and the above adds unnecessary newlines and reading from disk
+        return doc.file_data_text
 
 @click.command()
 @click.option(
@@ -80,19 +149,28 @@ def main(directory, user, password):
         click.echo("output directory does not exist, creating.")
         outpath.mkdir(parents=True)
 
-    notes = keep.all()
+    notes: List[gkeepapi.node.TopLevelNode] = keep.all()
     note_count = 0
-    for note in notes:
+    for note in notes: # type: gkeepapi.node.TopLevelNode
         note_count += 1
         click.echo(f"Processing note #{note_count}")
-        post = process_note(note)
+
+        title = note.title.strip()
+        if not len(title):
+            title = "untitled"
 
         outfile = (
             outpath
-            / f'{sanitize_filename(f"{note_count:04} - " + post.metadata["title"],max_len=135)} .md'
+            / f'{sanitize_filename(f"{note_count:04} - " + title,max_len=135)}.md'
         )
-        with outfile.open("wb") as fp:
-            frontmatter.dump(post, fp)
+
+        images = download_images(keep, note_count, note, outpath)
+        markdown = build_markdown(note, images)
+        fmatter = build_frontmatter(note, markdown)
+
+
+        with open(outfile, "wb+") as f:
+            frontmatter.dump(fmatter, f)
 
     click.echo("Done.")
 

poetry.lock

@@ -144,6 +144,14 @@ category = "dev"
 optional = false
 python-versions = "*"
 
+[[package]]
+name = "mdutils"
+version = "1.3.0"
+description = "Useful package for creating Markdown files while executing python code."
+category = "main"
+optional = false
+python-versions = "*"
+
 [[package]]
 name = "mypy-extensions"
 version = "0.4.3"
@@ -206,6 +214,7 @@ python-versions = "*"
 PyYAML = "*"
 six = "*"
 
+
 [[package]]
 name = "pyyaml"
 version = "5.3.1"
@@ -296,7 +305,7 @@ python-versions = "*"
 [metadata]
 lock-version = "1.1"
 python-versions = "^3.8"
-content-hash = "8db124cfd7ac8b010d8535c9faf5ca3f9e36d0b087ba2203d555b477c411bca8"
+content-hash = "608ac5f32f9eb09115d688a49cfd1ae71f81f09248caae13069940b9b7310f95"
 
 [metadata.files]
 appdirs = [
@@ -308,7 +317,6 @@ astroid = [
     {file = "astroid-2.4.2.tar.gz", hash = "sha256:2f4078c2a41bf377eea06d71c9d2ba4eb8f6b1af2135bec27bbbb7d8f12bb703"},
 ]
 black = [
-    {file = "black-20.8b1-py3-none-any.whl", hash = "sha256:70b62ef1527c950db59062cda342ea224d772abdf6adc58b86a45421bab20a6b"},
     {file = "black-20.8b1.tar.gz", hash = "sha256:1c02557aa099101b9d21496f8a914e9ed2222ef70336404eeeac8edba836fbea"},
 ]
 certifi = [
@@ -370,6 +378,9 @@ mccabe = [
     {file = "mccabe-0.6.1-py2.py3-none-any.whl", hash = "sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42"},
     {file = "mccabe-0.6.1.tar.gz", hash = "sha256:dd8d182285a0fe56bace7f45b5e7d1a6ebcbf524e8f3bd87eb0f125271b8831f"},
 ]
+mdutils = [
+    {file = "mdutils-1.3.0.tar.gz", hash = "sha256:098baa99cf80ccb1e98b051c3cda8f85486d4ccc9abeb8742c1cb3903c6862ae"},
+]
 mypy-extensions = [
     {file = "mypy_extensions-0.4.3-py2.py3-none-any.whl", hash = "sha256:090fedd75945a69ae91ce1303b5824f428daf5a028d2f6ab8a299250a846f15d"},
     {file = "mypy_extensions-0.4.3.tar.gz", hash = "sha256:2d82818f5bb3e369420cb3c4060a7970edba416647068eb4c5343488a6c604a8"},

pyproject.toml

@@ -15,6 +15,7 @@ python-frontmatter = "^0.5.0"
 PyYAML = "^5.3.1"
 pathvalidate = "^2.3.2"
 click = "^7.1.2"
+mdutils = "^1.3.0"
 
 [tool.poetry.dev-dependencies]
 black = {version = "^20.8b1", allow-prereleases = true}