podman-compose-tools/podman-compose-backup.py

#!/usr/bin/env python3


# TODO implement backup single vol
# TODO implement restore single vol
# TODO decide upon depends_on and volume mounts which containers must be shut down and which turned on for single volume backup (def)
# TODO group decision for multiple/all volumes (at first, throw error if incompatible as backed up state might be inconsistent)
# TODO store full backup (= all volumes) inside one (uncompressed) tar archive
# TODO support env-files
# TODO implement secrets
# TODO throw error/hint on bind mounts (not supported for now)

# TODO test with normal volume
# TODO test with mariadb
# TODO test with entertainment-decider (testing depends_on)
# TODO test with small nextcloud instance

# TODO append compose and referenced files into tar for easy migration
# TODO support for restoring from easy migration tar archive
# TODO support --podman-path / --podman-compose-path
# TODO support --podman-args
# TODO add profile support (if applicable), e.g. mysql/mariadb


# === imports


from __future__ import annotations

import argparse
from functools import cached_property, wraps
import os
from pathlib import Path, PurePath
import shutil
import sys
from typing import (
    Dict,
    Mapping,
    NewType,
    Optional,
    Sequence,
    TypeAlias,
    TypedDict,
    cast,
)

from attrs import define, field
from podman_compose import normalize, rec_merge, rec_subs
import yaml

from podman_compose_tools.defs.compose import (
    ComposeDef,
    ComposeVersion,
    ServiceName,
    ContainerName,
    ComposeServiceDef,
    ComposeServiceVolumeDef,
    VolumeName,
    PublicVolumeName,
    ComposeVolumeDef,
)
from podman_compose_tools.executor import (
    ArgCommand,
    BinaryExecutor,
    CompletedExec,
    ExecutorTarget,
    HostExecutor,
    ShellCommand,
)
from podman_compose_tools.executor.base import (
    combine_cmds,
    CommandArgs,
)


# === custom types


ProjectName = NewType("ProjectName", str)


LabelDict: TypeAlias = Mapping[str, str]


class VolumeInspectDef(TypedDict):
    Name: PublicVolumeName
    Driver: str
    Mountpoint: str
    CreatedAt: str  # ISO
    Labels: LabelDict
    Scope: str
    Options: Mapping


# === constants


# multiple prefixes for adding more if project becomes standardized
# first label takes precendence
LABEL_PREFIXES = [
    "work.banananet.podman.backup.",
]

DEFAULT_MOUNT_TARGET = "/_volume"
DEFAULT_BACKUP_IMAGE = "docker.io/library/debian:stable"
DEFAULT_BACKUP_CMD = "tar -cf - ."
DEFAULT_RESTORE_CMD = "tar -xf -"


PODMAN_EXEC = shutil.which("podman")
PODMAN_COMPOSE_EXEC = shutil.which("podman-compose")


# === helpers


host = HostExecutor()


@wraps(print)
def error(*args, **kwargs):
    ret = print(*args, file=sys.stderr, **kwargs)
    sys.stderr.flush()
    return ret


def parse_bool(val: str | bool) -> bool:
    if isinstance(val, bool):
        return val
    return val.lower().startswith(("t", "y", "1"))


# === code


# required because of mypy attrs converter restrictions
def shell_cmd_from_str(command: str) -> ShellCommand:
    return ShellCommand.from_str(command=command)


@define(kw_only=True)
class VolumeBackupConfig:
    # === Backups
    enable: bool = field(converter=parse_bool, default=True)
    container: Optional[str] = field(default=None)
    image: str = field(default=DEFAULT_BACKUP_IMAGE)
    mount_target: str = field(default=DEFAULT_MOUNT_TARGET)
    stop: bool = field(converter=parse_bool, default=False)
    backup_cmd: ShellCommand = field(
        converter=shell_cmd_from_str,
        default=shell_cmd_from_str(DEFAULT_BACKUP_CMD),
    )
    restore_cmd: ShellCommand = field(
        converter=shell_cmd_from_str,
        default=shell_cmd_from_str(DEFAULT_RESTORE_CMD),
    )
    # === Compressing
    compress_image: Optional[str] = field(default=None)
    compress_cmd: Optional[ShellCommand] = field(
        converter=shell_cmd_from_str,
        default=None,
    )
    decompress_cmd: Optional[ShellCommand] = field(
        converter=shell_cmd_from_str,
        default=None,
    )

    @classmethod
    def from_labels(cls, labels: LabelDict) -> VolumeBackupConfig:
        return cls(**parse_labels(labels=labels))

    def __attrs_post_init__(self):
        if self.compress_cmd is None:
            if self.decompress_cmd is not None:
                # TODO specialize
                raise Exception(
                    "compress-cmd must be specified as it cannot be retrieved from decompress-cmd"
                )
        else:
            self.decompress_cmd = self.compress_cmd + " -d"


@define(kw_only=True)
class PodmanClient:

    exec: BinaryExecutor = field(converter=lambda a: BinaryExecutor(a))
    compose_exec: BinaryExecutor = field(converter=lambda a: BinaryExecutor(a))


class ComposeFile(ExecutorTarget):

    podman: PodmanClient
    project_name: ProjectName
    environ: Dict[str, str]
    compose: ComposeDef
    compose_files: Sequence[Path]

    def __init__(
        self,
        podman: PodmanClient,
        *compose_files: Path,
        project_name: Optional[ProjectName] = None,
    ):
        self.podman = podman
        self.compose_files = compose_files
        ref_dir = compose_files[0].parent
        self.project_name = project_name or ProjectName(ref_dir.name)
        compose: ComposeDef = {
            "_dirname": ref_dir,
            "version": ComposeVersion("0"),
        }
        self.environ = dict(os.environ)
        for path in compose_files:
            with open(path, "r") as fh:
                content = yaml.safe_load(fh)
                if not isinstance(content, dict):
                    error(f"Compose file does not contain a top level object: {path}")
                    sys.exit(1)
                content = normalize(content)
                content = rec_subs(content, self.environ)
                rec_merge(compose, content)
        self.compose = compose
        if not self.version.startswith("3."):
            error(
                f"Compose file version is not supported, only support 3.X compose files"
            )
            sys.exit(1)

    @property
    def ref_dir(self) -> Path:
        return self.compose["_dirname"]

    @property
    def version(self) -> ComposeVersion:
        return self.compose.get("version", ComposeVersion(""))

    @property
    def __services_defs(self) -> Mapping[ServiceName, ComposeServiceDef]:
        return self.compose.get("services", {})

    @property
    def __volumes_defs(self) -> Mapping[VolumeName, Optional[ComposeVolumeDef]]:
        return self.compose.get("volumes", {})

    @cached_property
    def services(self) -> Mapping[ServiceName, ComposeService]:
        return {
            name: ComposeService(compose=self, name=name, base=opts or {})
            for name, opts in self.__services_defs.items()
        }

    @cached_property
    def volumes(self) -> Mapping[VolumeName, ComposeVolume]:
        return {
            name: ComposeVolume(compose=self, name=name, base=opts or {})
            for name, opts in self.__volumes_defs.items()
        }

    def exec_cmd(
        self,
        *,
        command: CommandArgs,
        check: bool = True,
        capture_stdout: bool = False,
        work_dir: Optional[PurePath] = None,
    ) -> CompletedExec:
        return super().exec_cmd(
            command=combine_cmds(
                [f"--project-name=self.project_name"],
                [f"--file={file}" for file in self.compose_files],
                command,
            ),
            check=check,
            capture_stdout=capture_stdout,
            work_dir=work_dir or self.ref_dir,
        )


@define(kw_only=True)
class ComposeService(ExecutorTarget):

    compose: ComposeFile
    name: ServiceName
    base: ComposeServiceDef

    @property
    def container_name(self) -> ContainerName:
        return self.base.get(
            "container_name",
            ContainerName(f"{self.compose.project_name}_{self.name}_1"),
        )

    @cached_property
    def depends_on(self) -> Sequence[ComposeService]:
        return [self.compose.services[name] for name in self.base.get("depends_on", [])]

    @cached_property
    def volume_mounts(self) -> Sequence[ComposeServiceVolume]:
        return [
            ComposeServiceVolume(service=self, volume_def=volume_def)
            for volume_def in self.base.get("volumes", [])
        ]

    def exec_cmd(
        self,
        command: CommandArgs,
        check: bool = True,
        capture_stdout: bool = False,
        work_dir: Optional[PurePath] = None,
    ) -> CompletedExec:
        return self.compose.podman.exec.exec_cmd(
            command=combine_cmds(
                [
                    "container",
                    "exec",
                    "--interactive=false",
                    None if work_dir is None else f"--workdir={work_dir}",
                    self.container_name,
                ],
                command,
            ),
            check=check,
            capture_stdout=capture_stdout,
        )


@define(kw_only=True)
class ComposeVolume:

    compose: ComposeFile
    name: VolumeName
    base: ComposeVolumeDef

    @cached_property
    def public_name(self) -> PublicVolumeName:
        return self.base.get(
            "name", PublicVolumeName(f"{self.compose.project_name}_{self.name}")
        )

    @cached_property
    def used_by(self) -> Sequence[ComposeServiceVolume]:
        return [
            service_vol
            for service in self.compose.services.values()
            for service_vol in service.volume_mounts
        ]

    @cached_property
    def backup_config(self) -> VolumeBackupConfig:
        return VolumeBackupConfig.from_labels(self.inspect()["Labels"])

    def inspect(self) -> VolumeInspectDef:
        return cast(
            VolumeInspectDef,
            self.compose.podman.exec.exec_cmd(
                command=CommandArgs(
                    [
                        "volume",
                        "inspect",
                        self.public_name,
                    ]
                ),
                check=True,
            ).to_json(),
        )


class ComposeServiceVolume:

    service: ComposeService
    volume: ComposeVolume
    read_only: bool

    def __init__(
        self,
        *,
        service: ComposeService,
        volume_def: ComposeServiceVolumeDef,
    ) -> None:
        self.service = service
        vol_name: VolumeName
        if isinstance(volume_def, str):
            values = volume_def.split(sep=":", maxsplit=2)
            if len(values) == 1:
                # implicit volume
                # TODO specialize
                raise Exception(f"Do not support implicit volumes: {volume_def!r}")
            if len(values) == 2:
                src, _ = values
                mode = "rw"  # default
            else:
                src, _, mode = values
            if mode not in {"ro", "rw"}:
                # TODO specialize
                raise Exception(f"Unsupported mode {mode!r} for volume {volume_def!r}")
            if "/" in src:
                # volume type: bind
                # TODO specialize
                raise Exception(
                    f"Unsupported volume type 'bind' for volume {volume_def!r}"
                )
            # volume type: volume
            vol_name = VolumeName(src)
            self.read_only = mode == "ro"
        else:
            if volume_def["type"] != "volume":
                # TODO specialize
                raise Exception(
                    f"Unsupported volume type {volume_def['type']!r} for volume"
                )
            vol_name = volume_def["source"]
            self.read_only = volume_def.get("read_only", False)
        self.volume = service.compose.volumes[vol_name]


def parse_labels(labels: LabelDict) -> LabelDict:
    ret = dict[str, str]()
    for key, val in labels.items():
        for prefix in LABEL_PREFIXES:
            if key.startswith(prefix):
                new_key = key.removeprefix(prefix).replace("-", "_")
                ret[new_key] = val
                break
    return ret


def parse_args(args: Sequence[str]) -> argparse.Namespace:
    parser = argparse.ArgumentParser(
        prog="podman-compose-backup",
        description="Backups & restores compose volumes according to their configuration",
    )
    parser.add_argument(
        "-f",
        "--file",
        nargs="*",
        type=Path,
        default=(Path("./docker-compose.yml"),),
        help="Specify an alternate compose file (default: docker-compose.yml)",
    )
    parser.add_argument(
        "-p",
        "--project-name",
        type=ProjectName,
        default=None,
        help="Specify an alternate project name (default: directory name)",
    )
    return parser.parse_args(args=args)


def exec(given_args: Sequence[str]):
    args = parse_args(args=given_args)
    compose = ComposeFile(*args.file, project_name=args.project_name)


def cli(args: Sequence[str]):
    try:
        exec(given_args=args)
    except (FileNotFoundError, IsADirectoryError, NotADirectoryError) as e:
        error(f"{e.strerror}: {e.filename}")
        sys.exit(2)


if __name__ == "__main__":
    cli(sys.argv[1:])