From 2d4db097530b95b9631f6009c5c37c9fe65166c8 Mon Sep 17 00:00:00 2001 From: Emilia Allison Date: Sun, 27 Oct 2024 23:07:44 -0400 Subject: [PATCH] Minimum working guy --- src/deploy/__init__.py | 0 src/deploy/_logging_formatter.py | 25 ++++++ src/deploy/_send.py | 109 ++++++++++++++++++++++++++ src/deploy/_staging.py | 80 +++++++++++++++++++ src/deploy/deploy.py | 62 +++++++++++++++ src/main.py | 4 + src/pictures/_util.py | 4 +- src/pictures/make_index.py | 1 + src/static_gen/templates/section.html | 2 +- 9 files changed, 284 insertions(+), 3 deletions(-) create mode 100644 src/deploy/__init__.py create mode 100644 src/deploy/_logging_formatter.py create mode 100644 src/deploy/_send.py create mode 100644 src/deploy/_staging.py create mode 100644 src/deploy/deploy.py diff --git a/src/deploy/__init__.py b/src/deploy/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/deploy/_logging_formatter.py b/src/deploy/_logging_formatter.py new file mode 100644 index 0000000..9bda101 --- /dev/null +++ b/src/deploy/_logging_formatter.py @@ -0,0 +1,25 @@ +import logging + + +class CustomFormatter(logging.Formatter): + + grey = "\x1b[38;20m" + yellow = "\x1b[33;20m" + red = "\x1b[31;20m" + bold_red = "\x1b[31;1m" + reset = "\x1b[0m" + format = "%(name)s-%(levelname)s: %(message)s" + + FORMATS = { + logging.DEBUG: grey + format + reset, + logging.INFO: grey + format + reset, + logging.WARNING: yellow + format + reset, + logging.ERROR: red + format + reset, + logging.CRITICAL: bold_red + format + reset + } + + def format(self, record): + log_fmt = self.FORMATS.get(record.levelno) + formatter = logging.Formatter(log_fmt) + record.msg = record.msg.replace("\t", " ") + return formatter.format(record) diff --git a/src/deploy/_send.py b/src/deploy/_send.py new file mode 100644 index 0000000..c2b7b47 --- /dev/null +++ b/src/deploy/_send.py @@ -0,0 +1,109 @@ +import logging + +from tempfile import NamedTemporaryFile +from subprocess import run, CalledProcessError, Popen, PIPE + +logger = logging.getLogger(__name__) + + +def send(ssh_host: str, + target_dir: str, + temp_archive: NamedTemporaryFile): + """Sends a temporary archive to a server, then runs deployment there. + """ + logger.info("Send started:") + transfer_file(ssh_host, "temp_pictures", temp_archive) + temp_archive.close() + ssh_process = spawn_ssh_process(ssh_host) + unpack(ssh_process, "temp_pictures") + move_to_final(ssh_process, target_dir, "temp_pictures") + logger.info("Send complete.\n") + + +def transfer_file(ssh_host: str, + target_temp_dir: str, + temp_archive: NamedTemporaryFile): + source = temp_archive.name + dest = f"{ssh_host}:{target_temp_dir}" + print(dest) + + logger.debug(f"\t\t Running: rsync ... {source} {dest}") + + logger.info("\t- Initiating file transfer") + try: + run(["/usr/bin/rsync", "-rt", "-e", "ssh", source, dest], + capture_output=True, + check=True) + except CalledProcessError: + transfer_recovery(target_temp_dir, temp_archive) + return + logger.info("\t\t File transfer complete") + + +def transfer_recovery(target_temp_dir: str, + temp_archive: NamedTemporaryFile): + logger.error("\tTransfer failed. Attempt recovery? (y/n)") + if input().upper() != "Y": + from sys import exit + logger.fatal("\tRecovery denied! ABORT") + exit(1) + logger.error(f"Local file: {temp_archive.name}\n" + + f"Remote file: {target_temp_dir}\n" + + "Hit [return] when transfer is confirmed.") + input() + logger.warn("Transfer recovery complete, continuing.") + + +def spawn_ssh_process(ssh_host: str) -> Popen: + from time import sleep + process = Popen(["ssh", "-tt", ssh_host], + stdin=PIPE, + stdout=PIPE, + bufsize=0, + text=True) + + # Hack to wait for user to enter password + process.stdout.readline() + + return process + + +def unpack(ssh_process: Popen, + target_temp_dir: str): + from multiprocessing import Process + + ssh_process.stdin.write(f"rm -rf {target_temp_dir}_dir \n") + ssh_process.stdin.write( + f"7z x -o{target_temp_dir}_dir {target_temp_dir} \n") + ssh_process.stdin.flush() + + def wait_for_done(ssh_process): + while True: + line = ssh_process.stdout.readline() + if "Everything is Ok" in line: + return + + p = Process(target=wait_for_done, args=(ssh_process, )) + p.start() + p.join(timeout=20) + + if p.is_alive(): + logger.error("Unpack timed out") + p.kill() + exit(1) + + +def move_to_final(ssh_process: Popen, + target_dir: str, + target_temp_dir: str): + logger.warn(f"About to delete {target_dir}, ok? (y/n)") + if input().upper() != "Y": + logger.fatal("Fail to confirm file deletion, ABORT") + exit(1) + ssh_process.stdin.write(f"rm -rf {target_dir} \n") + ssh_process.stdin.write(f"cp -r {target_temp_dir}_dir {target_dir} \n") + ssh_process.stdin.write(f"chown -R ubuntu:www {target_dir} \n") + ssh_process.stdin.write(f"chmod -R 707 {target_dir} \n") + ssh_process.stdin.write(f"rm -rf {target_temp_dir}" + + f" {target_temp_dir}_dir \n") + ssh_process.stdin.write("bye") diff --git a/src/deploy/_staging.py b/src/deploy/_staging.py new file mode 100644 index 0000000..1775ad0 --- /dev/null +++ b/src/deploy/_staging.py @@ -0,0 +1,80 @@ +import logging + +from tempfile import TemporaryDirectory, NamedTemporaryFile +from os.path import join, basename, dirname, relpath +from pictures._util import check_valid_archive + +logger = logging.getLogger(__name__) + + +def stage(archive_path: str) -> NamedTemporaryFile: + if not check_valid_archive(archive_path): + raise ValueError("Archive not valid. Refusing to deploy.") + + logger.info("Staging started:") + + logger.info("\t- Creating a temporary directory") + with TemporaryDirectory() as dir: + unpack(dir, archive_path) + render(dir, archive_path) + temp = compress(dir) + logger.info("Staging complete.\n") + return temp + + +def unpack(dir: str, archive_path: str): + from pictures._util import check_jpg + from zipfile import ZipFile + + logger.info("\t- Unpacking images") + with ZipFile(archive_path, 'r') as zip: + required_files = [f for f in zip.namelist() if check_jpg(f)] + for file in required_files: + logger.debug(f"Extracting file: {file}") + zip.extract(file, path=dir) + logger.info("\t\tImages successfully unpacked.") + + +def render(dir: str, archive_path: str): + from pictures.make_index import make_index + from static_gen.render import gen_page + + logger.info("\t- Rendering HTML") + + logger.debug("\t\t INDEX GENERATION") + index = make_index(archive_path, False) + + logger.debug("\t\t RENDERING") + page = gen_page(index) + + logger.debug("\t\t WRITING OUTPUT") + with open(join(dir, "index.html"), 'w') as f: + f.write(page) + + logger.info("\t\t Rendering complete") + + +def compress(dir: str) -> NamedTemporaryFile: + from zipfile import ZipFile, ZIP_LZMA + from glob import glob + + logger.info("\t- Compressing staged directory") + + temp = NamedTemporaryFile(mode='w+b') + with ZipFile(temp, 'w', ZIP_LZMA) as zip: + logger.info("\t\t Writing index.html") + zip.write(join(dir, "index.html"), arcname="index.html") + + logger.info("\t\t Writing image files") + zip.mkdir("img") + for image in glob(f"{dir}/**/*.jpg", recursive=True) + \ + glob(f"{dir}/**/*.jpeg", recursive=True): + corrected_path = join("img", + relpath(image, dir)) + logger.debug(f"\t\tWriting: {corrected_path}") + zip.write(image, arcname=corrected_path) + + temp.seek(0) # Reset file pointer + logger.info("\t\t Compression complete.") + + return temp diff --git a/src/deploy/deploy.py b/src/deploy/deploy.py new file mode 100644 index 0000000..3899061 --- /dev/null +++ b/src/deploy/deploy.py @@ -0,0 +1,62 @@ +import logging +import sys +import argparse +from tempfile import NamedTemporaryFile + +from deploy._staging import stage +from deploy._send import send +from deploy._logging_formatter import CustomFormatter + +logger = logging.getLogger() + + +def deploy(ssh_host: str, + target_dir: str, + archive_path: str, + require_confirmation: bool = True): + """ Runs the deployment pipeline from start to finish. + + Args: + ssh_host: Exact string that will be used by ssh and rsync as the host. + target_dir: Exact, absolute directory where files will be placed; + anything already present will be clobbered. + archive_path: Path to the local archive to deploy from + Returns: + Nothing. + """ + logger.setLevel(logging.INFO) + ch = logging.StreamHandler(sys.stdout) + ch.setFormatter(CustomFormatter()) + ch.setLevel(logging.INFO) + logger.addHandler(ch) + + temp_archive: NamedTemporaryFile = stage(archive_path) + confirm_archive(temp_archive, require_confirmation) + send(ssh_host, target_dir, temp_archive) + + +def confirm_archive(temp_archive: NamedTemporaryFile, + require_confirmation: bool): + if require_confirmation: + logging.warn(f"Temp archive currently exists at: {temp_archive.name}") + confirm = input("Please confirm archive validity. (y/n)") + if confirm.upper() != "Y": + logging.fatal("CONFIRMATION FAILED, ABORT") + temp_archive.close() + sys.exit(1) + else: + logging.warn("Confirmation skipped.") + + +def add_deploy_command(top_level_parsers: argparse._SubParsersAction): + parser: argparse.ArgumentParser = top_level_parsers.add_parser("deploy") + parser.add_argument("-a", "--archive", + help="path to archive", + dest="archive_path", + nargs=1, + required=True) + parser.add_argument(dest="ssh_host") + parser.add_argument(dest="target_dir") + parser.set_defaults(func=lambda a: deploy(ssh_host=a.ssh_host, + target_dir=a.target_dir, + archive_path=a.archive_path[0])) diff --git a/src/main.py b/src/main.py index ea8d990..04b9df4 100755 --- a/src/main.py +++ b/src/main.py @@ -3,6 +3,7 @@ import argparse from archive_commands import add_archive_parser from render_command import add_render_parser +from deploy.deploy import add_deploy_command def main(): @@ -18,6 +19,9 @@ def main(): # Render : Create static site add_render_parser(subparsers) + # Deploy + add_deploy_command(subparsers) + # Run args = parser.parse_args() args.func(args) diff --git a/src/pictures/_util.py b/src/pictures/_util.py index 3669a15..d7b0a9a 100644 --- a/src/pictures/_util.py +++ b/src/pictures/_util.py @@ -1,7 +1,7 @@ # Util functions for picture archive from zipfile import ZipFile, is_zipfile, ZIP_LZMA -from os.path import isfile, basename, splittext +from os.path import isfile, basename, splitext ARCHIVE_MARKER_FILE = "PICARCHIVE" @@ -47,7 +47,7 @@ def get_already_existing_files(path: str, files: [str]) -> [str]: def check_ext(path: str, valid: [str]) -> bool: - return splittext(path)[1].upper() in [x.upper() for x in valid] + return splitext(path)[1].upper() in [x.upper() for x in valid] def check_jpg(path: str) -> bool: diff --git a/src/pictures/make_index.py b/src/pictures/make_index.py index d6384ae..9115cd1 100644 --- a/src/pictures/make_index.py +++ b/src/pictures/make_index.py @@ -17,6 +17,7 @@ def make_index(archive_path: str, for y in names if dirname(y) == x and check_jpg(y)]) for x in section_names] + sections = [x for x in sections if len(x.images) > 0] if search_captions: raise NotImplementedError diff --git a/src/static_gen/templates/section.html b/src/static_gen/templates/section.html index 88832f3..ae0923d 100644 --- a/src/static_gen/templates/section.html +++ b/src/static_gen/templates/section.html @@ -4,7 +4,7 @@
{% for image in section.images -%}
- + {%- if image.caption is not none -%}
{{ image.caption }}
{%- endif %}