Minimum working guy

src/deploy/_logging_formatter.py

@@ -0,0 +1,25 @@
+import logging
+
+
+class CustomFormatter(logging.Formatter):
+
+    grey = "\x1b[38;20m"
+    yellow = "\x1b[33;20m"
+    red = "\x1b[31;20m"
+    bold_red = "\x1b[31;1m"
+    reset = "\x1b[0m"
+    format = "%(name)s-%(levelname)s: %(message)s"
+
+    FORMATS = {
+        logging.DEBUG: grey + format + reset,
+        logging.INFO: grey + format + reset,
+        logging.WARNING: yellow + format + reset,
+        logging.ERROR: red + format + reset,
+        logging.CRITICAL: bold_red + format + reset
+    }
+
+    def format(self, record):
+        log_fmt = self.FORMATS.get(record.levelno)
+        formatter = logging.Formatter(log_fmt)
+        record.msg = record.msg.replace("\t", "  ")
+        return formatter.format(record)

src/deploy/_send.py

@@ -0,0 +1,109 @@
+import logging
+
+from tempfile import NamedTemporaryFile
+from subprocess import run, CalledProcessError, Popen, PIPE
+
+logger = logging.getLogger(__name__)
+
+
+def send(ssh_host: str,
+         target_dir: str,
+         temp_archive: NamedTemporaryFile):
+    """Sends a temporary archive to a server, then runs deployment there.
+    """
+    logger.info("Send started:")
+    transfer_file(ssh_host, "temp_pictures", temp_archive)
+    temp_archive.close()
+    ssh_process = spawn_ssh_process(ssh_host)
+    unpack(ssh_process, "temp_pictures")
+    move_to_final(ssh_process, target_dir, "temp_pictures")
+    logger.info("Send complete.\n")
+
+
+def transfer_file(ssh_host: str,
+                  target_temp_dir: str,
+                  temp_archive: NamedTemporaryFile):
+    source = temp_archive.name
+    dest = f"{ssh_host}:{target_temp_dir}"
+    print(dest)
+
+    logger.debug(f"\t\t Running: rsync ... {source} {dest}")
+
+    logger.info("\t- Initiating file transfer")
+    try:
+        run(["/usr/bin/rsync", "-rt", "-e", "ssh", source, dest],
+            capture_output=True,
+            check=True)
+    except CalledProcessError:
+        transfer_recovery(target_temp_dir, temp_archive)
+        return
+    logger.info("\t\t File transfer complete")
+
+
+def transfer_recovery(target_temp_dir: str,
+                      temp_archive: NamedTemporaryFile):
+    logger.error("\tTransfer failed. Attempt recovery? (y/n)")
+    if input().upper() != "Y":
+        from sys import exit
+        logger.fatal("\tRecovery denied! ABORT")
+        exit(1)
+    logger.error(f"Local file: {temp_archive.name}\n" +
+                 f"Remote file: {target_temp_dir}\n" +
+                 "Hit [return] when transfer is confirmed.")
+    input()
+    logger.warn("Transfer recovery complete, continuing.")
+
+
+def spawn_ssh_process(ssh_host: str) -> Popen:
+    from time import sleep
+    process = Popen(["ssh", "-tt", ssh_host],
+                    stdin=PIPE,
+                    stdout=PIPE,
+                    bufsize=0,
+                    text=True)
+
+    # Hack to wait for user to enter password
+    process.stdout.readline()
+
+    return process
+
+
+def unpack(ssh_process: Popen,
+           target_temp_dir: str):
+    from multiprocessing import Process
+
+    ssh_process.stdin.write(f"rm -rf {target_temp_dir}_dir \n")
+    ssh_process.stdin.write(
+        f"7z x -o{target_temp_dir}_dir {target_temp_dir} \n")
+    ssh_process.stdin.flush()
+
+    def wait_for_done(ssh_process):
+        while True:
+            line = ssh_process.stdout.readline()
+            if "Everything is Ok" in line:
+                return
+
+    p = Process(target=wait_for_done, args=(ssh_process, ))
+    p.start()
+    p.join(timeout=20)
+
+    if p.is_alive():
+        logger.error("Unpack timed out")
+        p.kill()
+        exit(1)
+
+
+def move_to_final(ssh_process: Popen,
+                  target_dir: str,
+                  target_temp_dir: str):
+    logger.warn(f"About to delete {target_dir}, ok? (y/n)")
+    if input().upper() != "Y":
+        logger.fatal("Fail to confirm file deletion, ABORT")
+        exit(1)
+    ssh_process.stdin.write(f"rm -rf {target_dir} \n")
+    ssh_process.stdin.write(f"cp -r {target_temp_dir}_dir {target_dir} \n")
+    ssh_process.stdin.write(f"chown -R ubuntu:www {target_dir} \n")
+    ssh_process.stdin.write(f"chmod -R 707 {target_dir} \n")
+    ssh_process.stdin.write(f"rm -rf {target_temp_dir}" +
+                            f" {target_temp_dir}_dir \n")
+    ssh_process.stdin.write("bye")

src/deploy/_staging.py

@@ -0,0 +1,80 @@
+import logging
+
+from tempfile import TemporaryDirectory, NamedTemporaryFile
+from os.path import join, basename, dirname, relpath
+from pictures._util import check_valid_archive
+
+logger = logging.getLogger(__name__)
+
+
+def stage(archive_path: str) -> NamedTemporaryFile:
+    if not check_valid_archive(archive_path):
+        raise ValueError("Archive not valid. Refusing to deploy.")
+
+    logger.info("Staging started:")
+
+    logger.info("\t- Creating a temporary directory")
+    with TemporaryDirectory() as dir:
+        unpack(dir, archive_path)
+        render(dir, archive_path)
+        temp = compress(dir)
+    logger.info("Staging complete.\n")
+    return temp
+
+
+def unpack(dir: str, archive_path: str):
+    from pictures._util import check_jpg
+    from zipfile import ZipFile
+
+    logger.info("\t- Unpacking images")
+    with ZipFile(archive_path, 'r') as zip:
+        required_files = [f for f in zip.namelist() if check_jpg(f)]
+        for file in required_files:
+            logger.debug(f"Extracting file: {file}")
+            zip.extract(file, path=dir)
+    logger.info("\t\tImages successfully unpacked.")
+
+
+def render(dir: str, archive_path: str):
+    from pictures.make_index import make_index
+    from static_gen.render import gen_page
+
+    logger.info("\t- Rendering HTML")
+
+    logger.debug("\t\t INDEX GENERATION")
+    index = make_index(archive_path, False)
+
+    logger.debug("\t\t RENDERING")
+    page = gen_page(index)
+
+    logger.debug("\t\t WRITING OUTPUT")
+    with open(join(dir, "index.html"), 'w') as f:
+        f.write(page)
+
+    logger.info("\t\t Rendering complete")
+
+
+def compress(dir: str) -> NamedTemporaryFile:
+    from zipfile import ZipFile, ZIP_LZMA
+    from glob import glob
+
+    logger.info("\t- Compressing staged directory")
+
+    temp = NamedTemporaryFile(mode='w+b')
+    with ZipFile(temp, 'w', ZIP_LZMA) as zip:
+        logger.info("\t\t Writing index.html")
+        zip.write(join(dir, "index.html"), arcname="index.html")
+
+        logger.info("\t\t Writing image files")
+        zip.mkdir("img")
+        for image in glob(f"{dir}/**/*.jpg", recursive=True) + \
+                glob(f"{dir}/**/*.jpeg", recursive=True):
+            corrected_path = join("img",
+                                  relpath(image, dir))
+            logger.debug(f"\t\tWriting: {corrected_path}")
+            zip.write(image, arcname=corrected_path)
+
+    temp.seek(0)  # Reset file pointer
+    logger.info("\t\t Compression complete.")
+
+    return temp

src/deploy/deploy.py

@@ -0,0 +1,62 @@
+import logging
+import sys
+import argparse
+from tempfile import NamedTemporaryFile
+
+from deploy._staging import stage
+from deploy._send import send
+from deploy._logging_formatter import CustomFormatter
+
+logger = logging.getLogger()
+
+
+def deploy(ssh_host: str,
+           target_dir: str,
+           archive_path: str,
+           require_confirmation: bool = True):
+    """ Runs the deployment pipeline from start to finish.
+
+    Args:
+        ssh_host: Exact string that will be used by ssh and rsync as the host.
+        target_dir: Exact, absolute directory where files will be placed;
+                    anything already present will be clobbered.
+        archive_path: Path to the local archive to deploy from
+    Returns:
+        Nothing.
+    """
+    logger.setLevel(logging.INFO)
+    ch = logging.StreamHandler(sys.stdout)
+    ch.setFormatter(CustomFormatter())
+    ch.setLevel(logging.INFO)
+    logger.addHandler(ch)
+
+    temp_archive: NamedTemporaryFile = stage(archive_path)
+    confirm_archive(temp_archive, require_confirmation)
+    send(ssh_host, target_dir, temp_archive)
+
+
+def confirm_archive(temp_archive: NamedTemporaryFile,
+                    require_confirmation: bool):
+    if require_confirmation:
+        logging.warn(f"Temp archive currently exists at: {temp_archive.name}")
+        confirm = input("Please confirm archive validity. (y/n)")
+        if confirm.upper() != "Y":
+            logging.fatal("CONFIRMATION FAILED, ABORT")
+            temp_archive.close()
+            sys.exit(1)
+    else:
+        logging.warn("Confirmation skipped.")
+
+
+def add_deploy_command(top_level_parsers: argparse._SubParsersAction):
+    parser: argparse.ArgumentParser = top_level_parsers.add_parser("deploy")
+    parser.add_argument("-a", "--archive",
+                        help="path to archive",
+                        dest="archive_path",
+                        nargs=1,
+                        required=True)
+    parser.add_argument(dest="ssh_host")
+    parser.add_argument(dest="target_dir")
+    parser.set_defaults(func=lambda a: deploy(ssh_host=a.ssh_host,
+                                              target_dir=a.target_dir,
+                                              archive_path=a.archive_path[0]))

src/main.py

@@ -3,6 +3,7 @@
 import argparse
 from archive_commands import add_archive_parser
 from render_command import add_render_parser
+from deploy.deploy import add_deploy_command
 
 
 def main():
@@ -18,6 +19,9 @@ def main():
     # Render : Create static site
     add_render_parser(subparsers)
 
+    # Deploy
+    add_deploy_command(subparsers)
+
     # Run
     args = parser.parse_args()
     args.func(args)

src/pictures/_util.py

@@ -1,7 +1,7 @@
 # Util functions for picture archive
 
 from zipfile import ZipFile, is_zipfile, ZIP_LZMA
-from os.path import isfile, basename, splittext
+from os.path import isfile, basename, splitext
 
 
 ARCHIVE_MARKER_FILE = "PICARCHIVE"
@@ -47,7 +47,7 @@ def get_already_existing_files(path: str, files: [str]) -> [str]:
 
 
 def check_ext(path: str, valid: [str]) -> bool:
-    return splittext(path)[1].upper() in [x.upper() for x in valid]
+    return splitext(path)[1].upper() in [x.upper() for x in valid]
 
 
 def check_jpg(path: str) -> bool:

src/pictures/make_index.py

@@ -17,6 +17,7 @@ def make_index(archive_path: str,
                             for y in names
                             if dirname(y) == x and check_jpg(y)])
                 for x in section_names]
+    sections = [x for x in sections if len(x.images) > 0]
 
     if search_captions:
         raise NotImplementedError

src/static_gen/templates/section.html

@@ -4,7 +4,7 @@
 	<div class="gallery">
 		{% for image in section.images -%}
 		<figure>
-			<img src="{{image.src}}" />
+			<img src="img/{{image.src}}" />
 			{%- if image.caption is not none -%}
 			<figcaption>{{ image.caption }}</figcaption>
 			{%- endif %}