Minimum working guy
This commit is contained in:
parent
0e21ffe890
commit
2d4db09753
|
@ -0,0 +1,25 @@
|
|||
import logging
|
||||
|
||||
|
||||
class CustomFormatter(logging.Formatter):
|
||||
|
||||
grey = "\x1b[38;20m"
|
||||
yellow = "\x1b[33;20m"
|
||||
red = "\x1b[31;20m"
|
||||
bold_red = "\x1b[31;1m"
|
||||
reset = "\x1b[0m"
|
||||
format = "%(name)s-%(levelname)s: %(message)s"
|
||||
|
||||
FORMATS = {
|
||||
logging.DEBUG: grey + format + reset,
|
||||
logging.INFO: grey + format + reset,
|
||||
logging.WARNING: yellow + format + reset,
|
||||
logging.ERROR: red + format + reset,
|
||||
logging.CRITICAL: bold_red + format + reset
|
||||
}
|
||||
|
||||
def format(self, record):
|
||||
log_fmt = self.FORMATS.get(record.levelno)
|
||||
formatter = logging.Formatter(log_fmt)
|
||||
record.msg = record.msg.replace("\t", " ")
|
||||
return formatter.format(record)
|
|
@ -0,0 +1,109 @@
|
|||
import logging
|
||||
|
||||
from tempfile import NamedTemporaryFile
|
||||
from subprocess import run, CalledProcessError, Popen, PIPE
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def send(ssh_host: str,
|
||||
target_dir: str,
|
||||
temp_archive: NamedTemporaryFile):
|
||||
"""Sends a temporary archive to a server, then runs deployment there.
|
||||
"""
|
||||
logger.info("Send started:")
|
||||
transfer_file(ssh_host, "temp_pictures", temp_archive)
|
||||
temp_archive.close()
|
||||
ssh_process = spawn_ssh_process(ssh_host)
|
||||
unpack(ssh_process, "temp_pictures")
|
||||
move_to_final(ssh_process, target_dir, "temp_pictures")
|
||||
logger.info("Send complete.\n")
|
||||
|
||||
|
||||
def transfer_file(ssh_host: str,
|
||||
target_temp_dir: str,
|
||||
temp_archive: NamedTemporaryFile):
|
||||
source = temp_archive.name
|
||||
dest = f"{ssh_host}:{target_temp_dir}"
|
||||
print(dest)
|
||||
|
||||
logger.debug(f"\t\t Running: rsync ... {source} {dest}")
|
||||
|
||||
logger.info("\t- Initiating file transfer")
|
||||
try:
|
||||
run(["/usr/bin/rsync", "-rt", "-e", "ssh", source, dest],
|
||||
capture_output=True,
|
||||
check=True)
|
||||
except CalledProcessError:
|
||||
transfer_recovery(target_temp_dir, temp_archive)
|
||||
return
|
||||
logger.info("\t\t File transfer complete")
|
||||
|
||||
|
||||
def transfer_recovery(target_temp_dir: str,
|
||||
temp_archive: NamedTemporaryFile):
|
||||
logger.error("\tTransfer failed. Attempt recovery? (y/n)")
|
||||
if input().upper() != "Y":
|
||||
from sys import exit
|
||||
logger.fatal("\tRecovery denied! ABORT")
|
||||
exit(1)
|
||||
logger.error(f"Local file: {temp_archive.name}\n" +
|
||||
f"Remote file: {target_temp_dir}\n" +
|
||||
"Hit [return] when transfer is confirmed.")
|
||||
input()
|
||||
logger.warn("Transfer recovery complete, continuing.")
|
||||
|
||||
|
||||
def spawn_ssh_process(ssh_host: str) -> Popen:
|
||||
from time import sleep
|
||||
process = Popen(["ssh", "-tt", ssh_host],
|
||||
stdin=PIPE,
|
||||
stdout=PIPE,
|
||||
bufsize=0,
|
||||
text=True)
|
||||
|
||||
# Hack to wait for user to enter password
|
||||
process.stdout.readline()
|
||||
|
||||
return process
|
||||
|
||||
|
||||
def unpack(ssh_process: Popen,
|
||||
target_temp_dir: str):
|
||||
from multiprocessing import Process
|
||||
|
||||
ssh_process.stdin.write(f"rm -rf {target_temp_dir}_dir \n")
|
||||
ssh_process.stdin.write(
|
||||
f"7z x -o{target_temp_dir}_dir {target_temp_dir} \n")
|
||||
ssh_process.stdin.flush()
|
||||
|
||||
def wait_for_done(ssh_process):
|
||||
while True:
|
||||
line = ssh_process.stdout.readline()
|
||||
if "Everything is Ok" in line:
|
||||
return
|
||||
|
||||
p = Process(target=wait_for_done, args=(ssh_process, ))
|
||||
p.start()
|
||||
p.join(timeout=20)
|
||||
|
||||
if p.is_alive():
|
||||
logger.error("Unpack timed out")
|
||||
p.kill()
|
||||
exit(1)
|
||||
|
||||
|
||||
def move_to_final(ssh_process: Popen,
|
||||
target_dir: str,
|
||||
target_temp_dir: str):
|
||||
logger.warn(f"About to delete {target_dir}, ok? (y/n)")
|
||||
if input().upper() != "Y":
|
||||
logger.fatal("Fail to confirm file deletion, ABORT")
|
||||
exit(1)
|
||||
ssh_process.stdin.write(f"rm -rf {target_dir} \n")
|
||||
ssh_process.stdin.write(f"cp -r {target_temp_dir}_dir {target_dir} \n")
|
||||
ssh_process.stdin.write(f"chown -R ubuntu:www {target_dir} \n")
|
||||
ssh_process.stdin.write(f"chmod -R 707 {target_dir} \n")
|
||||
ssh_process.stdin.write(f"rm -rf {target_temp_dir}" +
|
||||
f" {target_temp_dir}_dir \n")
|
||||
ssh_process.stdin.write("bye")
|
|
@ -0,0 +1,80 @@
|
|||
import logging
|
||||
|
||||
from tempfile import TemporaryDirectory, NamedTemporaryFile
|
||||
from os.path import join, basename, dirname, relpath
|
||||
from pictures._util import check_valid_archive
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def stage(archive_path: str) -> NamedTemporaryFile:
|
||||
if not check_valid_archive(archive_path):
|
||||
raise ValueError("Archive not valid. Refusing to deploy.")
|
||||
|
||||
logger.info("Staging started:")
|
||||
|
||||
logger.info("\t- Creating a temporary directory")
|
||||
with TemporaryDirectory() as dir:
|
||||
unpack(dir, archive_path)
|
||||
render(dir, archive_path)
|
||||
temp = compress(dir)
|
||||
logger.info("Staging complete.\n")
|
||||
return temp
|
||||
|
||||
|
||||
def unpack(dir: str, archive_path: str):
|
||||
from pictures._util import check_jpg
|
||||
from zipfile import ZipFile
|
||||
|
||||
logger.info("\t- Unpacking images")
|
||||
with ZipFile(archive_path, 'r') as zip:
|
||||
required_files = [f for f in zip.namelist() if check_jpg(f)]
|
||||
for file in required_files:
|
||||
logger.debug(f"Extracting file: {file}")
|
||||
zip.extract(file, path=dir)
|
||||
logger.info("\t\tImages successfully unpacked.")
|
||||
|
||||
|
||||
def render(dir: str, archive_path: str):
|
||||
from pictures.make_index import make_index
|
||||
from static_gen.render import gen_page
|
||||
|
||||
logger.info("\t- Rendering HTML")
|
||||
|
||||
logger.debug("\t\t INDEX GENERATION")
|
||||
index = make_index(archive_path, False)
|
||||
|
||||
logger.debug("\t\t RENDERING")
|
||||
page = gen_page(index)
|
||||
|
||||
logger.debug("\t\t WRITING OUTPUT")
|
||||
with open(join(dir, "index.html"), 'w') as f:
|
||||
f.write(page)
|
||||
|
||||
logger.info("\t\t Rendering complete")
|
||||
|
||||
|
||||
def compress(dir: str) -> NamedTemporaryFile:
|
||||
from zipfile import ZipFile, ZIP_LZMA
|
||||
from glob import glob
|
||||
|
||||
logger.info("\t- Compressing staged directory")
|
||||
|
||||
temp = NamedTemporaryFile(mode='w+b')
|
||||
with ZipFile(temp, 'w', ZIP_LZMA) as zip:
|
||||
logger.info("\t\t Writing index.html")
|
||||
zip.write(join(dir, "index.html"), arcname="index.html")
|
||||
|
||||
logger.info("\t\t Writing image files")
|
||||
zip.mkdir("img")
|
||||
for image in glob(f"{dir}/**/*.jpg", recursive=True) + \
|
||||
glob(f"{dir}/**/*.jpeg", recursive=True):
|
||||
corrected_path = join("img",
|
||||
relpath(image, dir))
|
||||
logger.debug(f"\t\tWriting: {corrected_path}")
|
||||
zip.write(image, arcname=corrected_path)
|
||||
|
||||
temp.seek(0) # Reset file pointer
|
||||
logger.info("\t\t Compression complete.")
|
||||
|
||||
return temp
|
|
@ -0,0 +1,62 @@
|
|||
import logging
|
||||
import sys
|
||||
import argparse
|
||||
from tempfile import NamedTemporaryFile
|
||||
|
||||
from deploy._staging import stage
|
||||
from deploy._send import send
|
||||
from deploy._logging_formatter import CustomFormatter
|
||||
|
||||
logger = logging.getLogger()
|
||||
|
||||
|
||||
def deploy(ssh_host: str,
|
||||
target_dir: str,
|
||||
archive_path: str,
|
||||
require_confirmation: bool = True):
|
||||
""" Runs the deployment pipeline from start to finish.
|
||||
|
||||
Args:
|
||||
ssh_host: Exact string that will be used by ssh and rsync as the host.
|
||||
target_dir: Exact, absolute directory where files will be placed;
|
||||
anything already present will be clobbered.
|
||||
archive_path: Path to the local archive to deploy from
|
||||
Returns:
|
||||
Nothing.
|
||||
"""
|
||||
logger.setLevel(logging.INFO)
|
||||
ch = logging.StreamHandler(sys.stdout)
|
||||
ch.setFormatter(CustomFormatter())
|
||||
ch.setLevel(logging.INFO)
|
||||
logger.addHandler(ch)
|
||||
|
||||
temp_archive: NamedTemporaryFile = stage(archive_path)
|
||||
confirm_archive(temp_archive, require_confirmation)
|
||||
send(ssh_host, target_dir, temp_archive)
|
||||
|
||||
|
||||
def confirm_archive(temp_archive: NamedTemporaryFile,
|
||||
require_confirmation: bool):
|
||||
if require_confirmation:
|
||||
logging.warn(f"Temp archive currently exists at: {temp_archive.name}")
|
||||
confirm = input("Please confirm archive validity. (y/n)")
|
||||
if confirm.upper() != "Y":
|
||||
logging.fatal("CONFIRMATION FAILED, ABORT")
|
||||
temp_archive.close()
|
||||
sys.exit(1)
|
||||
else:
|
||||
logging.warn("Confirmation skipped.")
|
||||
|
||||
|
||||
def add_deploy_command(top_level_parsers: argparse._SubParsersAction):
|
||||
parser: argparse.ArgumentParser = top_level_parsers.add_parser("deploy")
|
||||
parser.add_argument("-a", "--archive",
|
||||
help="path to archive",
|
||||
dest="archive_path",
|
||||
nargs=1,
|
||||
required=True)
|
||||
parser.add_argument(dest="ssh_host")
|
||||
parser.add_argument(dest="target_dir")
|
||||
parser.set_defaults(func=lambda a: deploy(ssh_host=a.ssh_host,
|
||||
target_dir=a.target_dir,
|
||||
archive_path=a.archive_path[0]))
|
|
@ -3,6 +3,7 @@
|
|||
import argparse
|
||||
from archive_commands import add_archive_parser
|
||||
from render_command import add_render_parser
|
||||
from deploy.deploy import add_deploy_command
|
||||
|
||||
|
||||
def main():
|
||||
|
@ -18,6 +19,9 @@ def main():
|
|||
# Render : Create static site
|
||||
add_render_parser(subparsers)
|
||||
|
||||
# Deploy
|
||||
add_deploy_command(subparsers)
|
||||
|
||||
# Run
|
||||
args = parser.parse_args()
|
||||
args.func(args)
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
# Util functions for picture archive
|
||||
|
||||
from zipfile import ZipFile, is_zipfile, ZIP_LZMA
|
||||
from os.path import isfile, basename, splittext
|
||||
from os.path import isfile, basename, splitext
|
||||
|
||||
|
||||
ARCHIVE_MARKER_FILE = "PICARCHIVE"
|
||||
|
@ -47,7 +47,7 @@ def get_already_existing_files(path: str, files: [str]) -> [str]:
|
|||
|
||||
|
||||
def check_ext(path: str, valid: [str]) -> bool:
|
||||
return splittext(path)[1].upper() in [x.upper() for x in valid]
|
||||
return splitext(path)[1].upper() in [x.upper() for x in valid]
|
||||
|
||||
|
||||
def check_jpg(path: str) -> bool:
|
||||
|
|
|
@ -17,6 +17,7 @@ def make_index(archive_path: str,
|
|||
for y in names
|
||||
if dirname(y) == x and check_jpg(y)])
|
||||
for x in section_names]
|
||||
sections = [x for x in sections if len(x.images) > 0]
|
||||
|
||||
if search_captions:
|
||||
raise NotImplementedError
|
||||
|
|
|
@ -4,7 +4,7 @@
|
|||
<div class="gallery">
|
||||
{% for image in section.images -%}
|
||||
<figure>
|
||||
<img src="{{image.src}}" />
|
||||
<img src="img/{{image.src}}" />
|
||||
{%- if image.caption is not none -%}
|
||||
<figcaption>{{ image.caption }}</figcaption>
|
||||
{%- endif %}
|
||||
|
|
Loading…
Reference in New Issue