#!/usr/bin/env python3
# https://www.progressiverobot.com
# Copyright (c) 2026 Christopher Holloway / Progressive Robot Ltd
# SPDX-License-Identifier: AGPL-3.0-or-later

"""hmctl - administer an hMailServer over its REST API from a command line.

The Windows Control Panel is the administration program and always will be; on
Linux the administration is the browser Control Deck and the REST API. This is
the third way: the same API from a shell, for the things a shell is better at -
making fifty accounts from a file, reading a setting in a script, watching the
queue, asking what a rule criterion would decide, or pulling last month's
report out as a CSV.

It speaks nothing but HTTP and JSON, and it is one file with no dependencies
beyond the Python standard library, so it runs wherever Python 3.8 does -
including on the machine the server is on, where the .deb and the .rpm install
it as /usr/bin/hmctl.

CREDENTIALS, in the order they are looked for:

    --api-key KEY           or  HMAIL_API_KEY       Authorization: Bearer KEY
    --password PASSWORD     or  HMAIL_PASSWORD      Basic Administrator:PASSWORD
    --password-file FILE    or  HMAIL_PASSWORD_FILE the same, read from a file
    (nothing)                                       prompted for, not echoed

The server's address comes from --url or HMAIL_URL and defaults to
https://localhost:8045. A server with a self-signed certificate needs
--insecure, which is refused for any host but the loopback: a credential sent to
a remote host over a connection nobody checked is the failure this tool must not
make easy.

WHAT IT DOES NOT DO: it does not read or write the message store, and it is not
a backup tool of its own. `hmctl backup start` asks the server to run its own
backup, `hmctl backup restore` asks it to put one back, and `hmctl backup
passphrase` sets the passphrase its archives are encrypted with. A passphrase is
never taken as an argument - the process list is public - only from a file or
from the terminal.

Run `hmctl` with no arguments for the resources, `hmctl <resource> --help` for
one resource's verbs, and `hmctl api` for the escape hatch - any method, any
path - which is what to use when a route is newer than this file.
"""

import argparse
import base64
import csv
import getpass
import io
import json
import os
import ssl
import sys
import urllib.error
import urllib.parse
import urllib.request

# The server's version, which hmctl is released with: stamped at a release with the
# rest (RELEASE.md, "Version stamp"), and held to Common/Application/Version.h by
# build/check-cli-packages.py. Until 6.3.6 it said 1.0, its own first version.
VERSION = "6.3.6"

DEFAULT_URL = "https://localhost:8045"

EXIT_OK = 0
EXIT_ERROR = 1
EXIT_USAGE = 2
EXIT_AUTH = 3
EXIT_NOTFOUND = 4

# A refusal: the server understood, could have done it, and said no. It shares
# its number with EXIT_USAGE, because argparse exits 2 for a command line it
# cannot parse and that is not ours to change - and because the Linux server's
# own `hmailserver --sample-domain` answers 2 for the same thing, which is the
# number a script written against one of the two would already be reading.
EXIT_REFUSED = 2


# ---------------------------------------------------------------------------
# The resources.
#
# One table, because every resource answers the same four or five verbs over the
# same shapes and the differences between them are data, not code: the path, the
# name of the thing, what a listing is called in the answer, which fields are
# worth a column, and which verbs the API actually carries. A resource whose
# route does not exist is not listed here at all - `hmctl api` reaches it.
# ---------------------------------------------------------------------------

RESOURCES = {
    "domain": {
        "collection": "/api/v1/domains",
        "item": "/api/v1/domains/{id}",
        "key": "domains",
        "id": "name",
        "columns": ["name", "active", "max_size_mb", "account_count"],
        "verbs": ["list", "get", "create", "update", "delete"],
        "help": "the domains this server hosts",
        "create_example": "hmctl domain create --set name=example.com --set active=true",
    },
    "account": {
        # Listed and made under the domain that owns them, read and written at
        # their own address: which is how the API is shaped, and it is right -
        # an account belongs to a domain, and an address is unique in itself.
        "collection": "/api/v1/domains/{parent}/accounts",
        "item": "/api/v1/accounts/{id}",
        "parent": "domain",
        "key": "accounts",
        "id": "address",
        "columns": ["address", "active", "max_size_mb", "admin_level"],
        "verbs": ["list", "get", "create", "update", "delete"],
        "help": "the mailboxes",
        "create_example": "hmctl account create --domain example.com --set address=ann@example.com --set password=...",
    },
    "alias": {
        "collection": "/api/v1/domains/{parent}/aliases",
        "item": "/api/v1/aliases/{id}",
        "parent": "domain",
        "key": "aliases",
        "id": "address",
        "columns": ["name", "value", "active"],
        "verbs": ["list", "create", "delete"],
        "help": "address aliases",
        "create_example": "hmctl alias create --domain example.com --set name=sales@example.com --set value=ann@example.com",
    },
    "list": {
        "collection": "/api/v1/domains/{parent}/lists",
        "item": "/api/v1/lists/{id}",
        "parent": "domain",
        "key": "lists",
        "id": "address",
        "columns": ["address", "active", "mode", "member_count", "membership"],
        "verbs": ["list", "create", "update", "delete"],
        "help": "distribution lists",
    },
    "listmember": {
        # Who a list delivers to now: its own addresses and whatever its
        # membership query answers at the moment of the call, each row saying
        # which of the two it came from. Read-only, because there is nothing
        # here to write - the rows are the list's and the rest is the query's.
        "collection": "/api/v1/lists/{parent}/members",
        "parent": "address",
        "key": "members",
        "id": "address",
        "columns": ["address", "source"],
        "verbs": ["list"],
        "help": "who a distribution list delivers to now, rows and query together",
    },
    "suppression": {
        "collection": "/api/v1/domains/{parent}/suppressions",
        "item": "/api/v1/domains/{parent}/suppressions/{id}",
        "parent": "domain",
        "key": "items",
        "id": "address",
        "search": True,
        "columns": ["address", "category", "status", "source", "created", "expires"],
        "verbs": ["list", "create", "delete"],
        "help": "addresses a domain no longer sends to because mail to them bounced",
        "create_example": "hmctl suppression create --domain example.com --set address=gone@elsewhere.test --set-string note=\"old export\"",
    },
    "complaint": {
        "collection": "/api/v1/domains/{parent}/complaints",
        "parent": "domain",
        "key": "items",
        "id": "address",
        "search": True,
        "columns": ["address", "feedback_type", "sender", "subject", "provider", "arrival"],
        "verbs": ["list"],
        "help": "the abuse reports the providers' feedback loops have sent about a domain's mail",
    },
    "bounce": {
        "collection": "/api/v1/domains/{parent}/bounces",
        "parent": "domain",
        "key": "items",
        "id": "address",
        "search": True,
        "columns": ["address", "hard", "soft", "mailbox_full", "policy", "other", "consecutive_hard", "last_status", "suppressed"],
        "verbs": ["list"],
        "help": "the bounce counts a domain's suppression list is fed from",
    },
    "group": {
        "collection": "/api/v1/groups",
        "item": "/api/v1/groups/{id}",
        "key": "groups",
        "id": "id",
        "columns": ["id", "name"],
        "verbs": ["list", "get", "create", "update", "delete"],
        "help": "account groups, which a folder permission can name as one principal",
    },
    "rule": {
        "collection": "/api/v1/rules",
        "item": "/api/v1/rules/{id}",
        "key": "rules",
        "id": "id",
        "columns": ["id", "name", "active", "sort_order"],
        "verbs": ["list", "create", "update", "delete"],
        "help": "the global rules",
    },
    "route": {
        "collection": "/api/v1/routes",
        "item": "/api/v1/routes/{id}",
        "key": "routes",
        "id": "id",
        "columns": ["id", "domain_name", "target_smtp_host", "target_smtp_port"],
        "verbs": ["list", "create", "update", "delete"],
        "help": "SMTP routes",
    },
    "port": {
        "collection": "/api/v1/ports",
        "item": "/api/v1/ports/{id}",
        "key": "ports",
        "id": "id",
        "columns": ["id", "protocol", "address", "port", "connection_security"],
        "verbs": ["list", "create", "update", "delete"],
        "help": "the listeners",
    },
    "iprange": {
        "collection": "/api/v1/ipranges",
        "item": "/api/v1/ipranges/{id}",
        "key": "ipranges",
        "id": "id",
        "columns": ["id", "name", "lower_ip", "upper_ip", "priority"],
        "verbs": ["list", "create", "update", "delete"],
        "help": "IP ranges",
    },
    "certificate": {
        "collection": "/api/v1/certificates",
        "item": "/api/v1/certificates/{id}",
        "key": "certificates",
        "id": "id",
        "columns": ["id", "name", "certificate_file"],
        "verbs": ["list", "create", "delete"],
        "help": "SSL certificates",
    },
    "fetchaccount": {
        "collection": "/api/v1/accounts/{parent}/fetch-accounts",
        "item": "/api/v1/accounts/{parent}/fetch-accounts/{id}",
        "parent": "address",
        "key": "fetch_accounts",
        "id": "id",
        "columns": ["id", "name", "server_address", "active"],
        "verbs": ["list", "get", "create", "update", "delete"],
        "help": "external POP3 accounts an account collects from",
    },
    "webhook": {
        "collection": "/api/v1/webhooks",
        "item": "/api/v1/webhooks/{id}",
        "key": "webhooks",
        "id": "name",
        "columns": ["name", "url", "events", "domains", "enabled", "pending", "delivered", "dead_lettered"],
        "verbs": ["list", "get", "create", "update", "delete"],
        "help": "the outbound webhook subscriptions: where this server's mail events go",
        "create_example": "hmctl webhook create --set name=ticketing --set url=https://tickets.example/hook --set-string secret=... --set events=message.bounced",
    },
    "spamtrap": {
        # Listed and made under the domain that owns them, taken back at their
        # own address: an address is unique in itself and names the domain it
        # belongs to, which is the shape an alias has.
        "collection": "/api/v1/domains/{parent}/spam-traps",
        "item": "/api/v1/spam-traps/{id}",
        "parent": "domain",
        "key": "spam_traps",
        "id": "address",
        "columns": ["address", "created"],
        "verbs": ["list", "create", "delete"],
        "help": "spam traps: addresses of a domain that were never real",
        "create_example": "hmctl spamtrap create --domain example.com --set address=never-used@example.com",
    },
    "journalrule": {
        # Journaling and dual delivery. Server-wide, like the webhooks: a rule
        # names a domain, but the list is every domain's and one of them says
        # where another's mail goes.
        "collection": "/api/v1/journal-rules",
        "item": "/api/v1/journal-rules/{id}",
        "key": "journal_rules",
        "id": "id",
        "columns": ["id", "scope", "target", "direction", "mode", "journal_address", "enabled", "note"],
        "verbs": ["list", "create", "update", "delete"],
        "help": "journaling and dual delivery: a copy of a domain's or an account's mail, sent to an address",
        "create_example": "hmctl journalrule create --set target=example.com --set journal_address=archive@compliance.example --set mode=report",
    },
    "apikey": {
        "collection": "/api/v1/apikeys",
        "item": "/api/v1/apikeys/{id}",
        "key": "apikeys",
        "id": "id",
        "columns": ["id", "name", "scope", "domain", "created", "last_used"],
        "verbs": ["list", "create", "delete"],
        "help": "API keys (administrator password only)",
    },
}

SETTING_GROUPS = [
    "settings",
    "antispam",
    "antivirus",
    "logging",
    "backup",
    "cache",
    "indexing",
    "scripting",
    "bounces",
    "complaints",
    "push",
    "directories",
    "openpgp",
]


class ApiError(Exception):
    def __init__(self, status, message, body=None):
        Exception.__init__(self, message)
        self.status = status
        self.message = message
        self.body = body


class Client:
    """Everything that talks to the server. Nothing else in this file does."""

    def __init__(self, url, api_key=None, password=None, insecure=False, timeout=30):
        self.url = url.rstrip("/")
        self.api_key = api_key
        self.password = password
        self.timeout = timeout

        parsed = urllib.parse.urlparse(self.url)
        self.host = parsed.hostname or ""
        self.scheme = parsed.scheme

        if insecure:
            # A credential travelling to a host whose certificate nobody checked
            # is the mistake this tool must not make convenient. On loopback the
            # certificate is beside the point - nothing is on the wire - and a
            # self-signed one there is the ordinary case.
            if self.host not in ("localhost", "127.0.0.1", "::1", "[::1]"):
                raise ApiError(0, "--insecure is refused for %s: it is only allowed for the loopback address, "
                                  "because it would send the administrator password to a host nobody verified. "
                                  "Give the server a certificate the client trusts." % self.host)
            self.context = ssl._create_unverified_context()
        elif self.scheme == "https":
            self.context = ssl.create_default_context()
        else:
            self.context = None

    def authorization(self):
        if self.api_key:
            return "Bearer " + self.api_key
        if self.password is not None:
            raw = ("Administrator:" + self.password).encode("utf-8")
            return "Basic " + base64.b64encode(raw).decode("ascii")
        return None

    def request(self, method, path, body=None, query=None, headers=None):
        url = self.url + path
        if query:
            pairs = [(k, v) for k, v in query.items() if v is not None]
            if pairs:
                url += "?" + urllib.parse.urlencode(pairs)

        data = None
        extra = headers or {}
        headers = {"Accept": "application/json", "User-Agent": "hmctl/" + VERSION}
        # A caller's own header, for the one GET that carries a secret: the
        # archive listing's passphrase, which a GET has no body to put in.
        headers.update(extra)
        if body is not None:
            data = json.dumps(body).encode("utf-8")
            headers["Content-Type"] = "application/json"

        authorization = self.authorization()
        if authorization:
            headers["Authorization"] = authorization

        request = urllib.request.Request(url, data=data, headers=headers, method=method)

        try:
            if self.context is not None:
                response = urllib.request.urlopen(request, timeout=self.timeout, context=self.context)
            else:
                response = urllib.request.urlopen(request, timeout=self.timeout)
        except urllib.error.HTTPError as error:
            raw = error.read()
            message = None
            parsed = None
            try:
                parsed = json.loads(raw.decode("utf-8"))
                if isinstance(parsed, dict):
                    message = parsed.get("error") or parsed.get("message")
            except Exception:
                message = raw.decode("utf-8", "replace").strip() or None
            raise ApiError(error.code, message or ("HTTP %d" % error.code), parsed)
        except urllib.error.URLError as error:
            reason = getattr(error, "reason", error)
            if isinstance(reason, ssl.SSLError):
                raise ApiError(0, "TLS refused the server's certificate: %s. On loopback, --insecure accepts a "
                                  "self-signed one." % reason)
            raise ApiError(0, "cannot reach %s: %s" % (self.url, reason))

        with response:
            raw = response.read()
            if not raw:
                return None
            try:
                return json.loads(raw.decode("utf-8"))
            except ValueError:
                return raw.decode("utf-8", "replace")


# ---------------------------------------------------------------------------
# Output
# ---------------------------------------------------------------------------

def render(value, as_json, columns=None):
    if as_json:
        print(json.dumps(value, indent=2, sort_keys=True))
        return

    if value is None:
        return

    if isinstance(value, list):
        if not value:
            print("(none)")
            return
        if all(isinstance(row, dict) for row in value):
            print_table(value, columns)
            return
        for row in value:
            print(row)
        return

    if isinstance(value, dict):
        width = max(len(str(k)) for k in value)
        for key in sorted(value):
            item = value[key]
            if isinstance(item, (dict, list)):
                item = json.dumps(item, sort_keys=True)
            print("%-*s  %s" % (width, key, item))
        return

    print(value)


def print_table(rows, columns=None):
    if columns:
        present = [c for c in columns if any(c in row for row in rows)]
        extra = []
    else:
        present = []
        extra = []
    if not present:
        seen = []
        for row in rows:
            for key in row:
                if key not in seen:
                    seen.append(key)
        present = seen[:8]

    def cell(row, column):
        value = row.get(column, "")
        if isinstance(value, bool):
            return "yes" if value else "no"
        if value is None:
            return ""
        if isinstance(value, (dict, list)):
            return json.dumps(value, sort_keys=True)
        return str(value)

    widths = [len(c) for c in present]
    for row in rows:
        for index, column in enumerate(present):
            widths[index] = max(widths[index], len(cell(row, column)))

    print("  ".join(c.upper().ljust(widths[i]) for i, c in enumerate(present)).rstrip())
    print("  ".join("-" * widths[i] for i in range(len(present))))
    for row in rows:
        print("  ".join(cell(row, c).ljust(widths[i]) for i, c in enumerate(present)).rstrip())
    if extra:
        print("")


def collection(answer, key):
    """The rows out of an answer, whether it is a list or an object holding one."""
    if isinstance(answer, list):
        return answer
    if isinstance(answer, dict):
        if key in answer and isinstance(answer[key], list):
            return answer[key]
        for value in answer.values():
            if isinstance(value, list):
                return value
    return answer


def ini_rows(answer):
    """The rows of GET /api/v1/settings/ini: settings, each {name, value, secret}.

    The route answers names as well - the list alone, for the Linux test shim -
    and a server from before 15 September 2026 answered ONLY names. Until then
    this file read a settings array the route never sent, collection() fell
    through to the names, and `ini list` and `config export` were silently
    empty against every real server. A names-only answer is now turned into
    rows with no value, so that a list is a list of names rather than nothing.
    """
    rows = collection(answer, "settings")
    if not isinstance(rows, list):
        return []
    if rows and all(isinstance(row, str) for row in rows):
        return [{"name": row} for row in rows]
    return [row for row in rows if isinstance(row, dict)]


def parse_set(pairs):
    """--set name=value, with a value read as JSON when it looks like JSON.

    So --set active=true is a boolean and --set name=true is not a problem
    either: a string that must stay a string is written --set name:=true... no.
    Simpler and unambiguous: true, false, null, numbers and anything starting
    with { or [ are JSON; everything else is a string. A string that must hold
    the four letters t-r-u-e is written --set-string.
    """
    body = {}
    for pair in pairs or []:
        if "=" not in pair:
            raise ApiError(0, "--set takes name=value, not %r" % pair)
        name, _, value = pair.partition("=")
        name = name.strip()
        stripped = value.strip()
        if stripped in ("true", "false", "null") or stripped[:1] in "{[" or _is_number(stripped):
            try:
                body[name] = json.loads(stripped)
                continue
            except ValueError:
                pass
        body[name] = value
    return body


def _is_number(text):
    if not text:
        return False
    try:
        float(text)
        return True
    except ValueError:
        return False


def body_from(arguments):
    body = {}
    if getattr(arguments, "from_json", None):
        if arguments.from_json == "-":
            body.update(json.load(sys.stdin))
        else:
            with io.open(arguments.from_json, encoding="utf-8") as handle:
                body.update(json.load(handle))
    body.update(parse_set(getattr(arguments, "set", None)))
    for pair in getattr(arguments, "set_string", None) or []:
        name, _, value = pair.partition("=")
        body[name.strip()] = value
    return body


# ---------------------------------------------------------------------------
# The verbs
# ---------------------------------------------------------------------------

def resource_command(client, arguments):
    resource = RESOURCES[arguments.resource]
    verb = arguments.verb

    def collection_path():
        path = resource["collection"]
        if "{parent}" in path:
            parent = getattr(arguments, resource["parent"], None)
            if not parent:
                raise ApiError(0, "%s %s needs --%s: the API lists and creates them under it.%s" %
                               (arguments.resource, verb, resource["parent"],
                                ("\n  " + resource["create_example"]) if resource.get("create_example") else ""))
            path = path.replace("{parent}", urllib.parse.quote(parent, safe="@"))
        return path

    def item_path():
        path = resource["item"]
        if "{parent}" in path:
            parent = getattr(arguments, resource["parent"], None)
            if not parent:
                raise ApiError(0, "%s %s needs --%s" % (arguments.resource, verb, resource["parent"]))
            path = path.replace("{parent}", urllib.parse.quote(parent, safe="@"))
        return path.replace("{id}", urllib.parse.quote(str(arguments.id), safe="@"))

    if verb == "list":
        path = collection_path()
        if resource.get("search") and getattr(arguments, "search", None):
            path += "?search=" + urllib.parse.quote(arguments.search, safe="@")
        answer = client.request("GET", path)
        render(collection(answer, resource["key"]), arguments.json, resource["columns"])
        return EXIT_OK

    if verb == "get":
        render(client.request("GET", item_path()), arguments.json)
        return EXIT_OK

    if verb == "create":
        body = body_from(arguments)
        if not body:
            raise ApiError(0, "nothing to create: give --set name=value or --from-json FILE.%s" %
                           (("\n  " + resource["create_example"]) if resource.get("create_example") else ""))
        render(client.request("POST", collection_path(), body=body), arguments.json)
        return EXIT_OK

    if verb == "update":
        body = body_from(arguments)
        if not body:
            raise ApiError(0, "nothing to change: give --set name=value or --from-json FILE")
        render(client.request("PUT", item_path(), body=body), arguments.json)
        return EXIT_OK

    if verb == "delete":
        client.request("DELETE", item_path())
        if not arguments.json:
            print("%s %s deleted." % (arguments.resource, arguments.id))
        return EXIT_OK

    raise ApiError(0, "%s cannot %s" % (arguments.resource, verb))


# The levels held in a field the domain or the account already had, rather than
# in the overlay (GET /api/v1/settings/levels answers "column" for them), and the
# value that field holds for "take the level above" - what the routes read, since
# a null on such a field leaves it as it is. A field that is not here always
# holds a value of its own and cannot inherit: the digest's switch and hour, the
# two tags, the impersonation action, plus addressing and the country policy.
COLUMN_INHERIT = {
    "max_message_size_kb": 0,
    "session_idle_seconds": 0,
    "session_lifetime_seconds": 0,
    "session_one_per_device": 0,
    "session_bind_to_address": 0,
    "quota_warning_percent": -1,
    "quota_grace_percent": -1,
    "quota_send_stop_percent": -1,
    "message_retention_days": 0,
}

# The domain's two session switches are 0 (the server's), 1 (on) or 2 (off) in
# its own fields, where every other level says true or false.
SWITCH_COLUMNS = ("session_one_per_device", "session_bind_to_address")


def level_set_command(client, arguments):
    """hmctl domain set <domain> <name> <value|inherit>, and the same for an account.

    One value, three levels (docs/SettingLevels.md): the domain's or the account's
    own value of a setting that may be set below the server - the names GET
    /api/v1/settings/levels lists - or, with the word inherit, none of its own,
    so that it takes the level above. The value is read as --set reads one:
    true, false and numbers as such, anything else as a string. Where the level
    is a field the domain or the account already had, inherit is sent as that
    field's own "take the level above" (0 or -1), true and false as a domain
    session switch's 1 and 2, and inherit is refused for a field that has none.
    The domain's route requires active in every body, so it is read first and
    sent back as it is. What is in effect afterwards, and where it came from, is
    what is printed.
    """
    name = arguments.setting
    raw = arguments.value
    inherit = raw.strip().lower() == "inherit"

    # Where the level is held. A server from before the levels list answers
    # nothing here, and every name is taken as the overlay's, as it always was.
    storage = "overlay"
    try:
        levels = client.request("GET", "/api/v1/settings/levels") or {}
        for entry in levels.get("settings", []) if isinstance(levels, dict) else []:
            if entry.get("name") == name:
                storage = entry.get("domain_storage" if arguments.resource == "domain" else "account_storage", "overlay")
                break
    except ApiError as error:
        if error.status not in (403, 404):
            raise

    if storage == "column":
        if inherit:
            if name not in COLUMN_INHERIT:
                raise ApiError(400, "%s is the %s's own field and always holds a value of its own; there is no level above "
                               "for it to take. Set it to the value you want." % (name, arguments.resource))
            value = COLUMN_INHERIT[name]
        else:
            value = parse_set([name + "=" + raw])[name]
            if arguments.resource == "domain" and name in SWITCH_COLUMNS and isinstance(value, bool):
                value = 1 if value else 2
    else:
        value = None if inherit else parse_set([name + "=" + raw])[name]

    if arguments.resource == "domain":
        path = "/api/v1/domains/%s" % urllib.parse.quote(arguments.id, safe="")
        current = client.request("GET", path) or {}
        answer = client.request("PUT", path, body={"active": bool(current.get("active", True)), name: value}) or {}
    else:
        path = "/api/v1/accounts/%s" % urllib.parse.quote(arguments.id, safe="@")
        answer = client.request("PUT", path, body={name: value}) or {}
    if arguments.json:
        render(answer, True)
        return EXIT_OK
    effective = answer.get(name + "_effective")
    source = answer.get(name + "_source", "")
    print("%s is %s for %s, from the %s." % (name, json.dumps(effective), arguments.id, source or "server"))
    return EXIT_OK


def status_command(client, arguments):
    render(client.request("GET", "/api/v1/status"), arguments.json)
    return EXIT_OK


def queue_command(client, arguments):
    if arguments.verb == "list":
        # The route's own names. held is "sandbox" for a message waiting for a
        # sandbox's verdict, which retry and delete answer 409 for: the
        # sandbox's routes (/api/v1/sandbox/held) decide it.
        answer = client.request("GET", "/api/v1/queue")
        render(collection(answer, "messages"), arguments.json,
               ["id", "from", "recipients", "next_try", "tries", "held"])
    elif arguments.verb == "retry":
        client.request("POST", "/api/v1/queue/%s/retry" % arguments.id)
        print("queued for delivery now.")
    elif arguments.verb == "delete":
        client.request("DELETE", "/api/v1/queue/%s" % arguments.id)
        print("message %s deleted from the queue." % arguments.id)
    return EXIT_OK


def privacy_command(client, arguments):
    if arguments.verb == "export":
        path = "/api/v1/accounts/%s/export" % urllib.parse.quote(arguments.address, safe="")
        answer = client.request("POST", path, {"archive": arguments.archive}) or {}
        print("exported %s message(s), %s contact(s), %s calendar object(s) and %s log line(s) to %s."
              % (answer.get("exported", 0), answer.get("contacts", 0), answer.get("calendar_objects", 0),
                 answer.get("log_lines", 0), answer.get("archive", arguments.archive)))
    elif arguments.verb == "erase":
        body = {"address": arguments.address, "include_archive": bool(arguments.archive), "include_logs": bool(arguments.logs)}
        answer = client.request("POST", "/api/v1/addresses/erase", body) or {}
        print("erased %s trace(s) of %s." % (answer.get("removed", 0), arguments.address))
        if answer.get("held", 0):
            print("%s item(s) were left in place under a legal hold." % answer.get("held"))
    return EXIT_OK


def quarantine_command(client, arguments):
    if arguments.verb == "list":
        answer = client.request("GET", "/api/v1/quarantine")
        render(collection(answer, "messages"), arguments.json,
               ["id", "recipient", "from_address", "subject", "reason", "received"])
    elif arguments.verb == "release":
        client.request("POST", "/api/v1/quarantine/%s/release" % arguments.id)
        print("released.")
    elif arguments.verb == "delete":
        client.request("DELETE", "/api/v1/quarantine/%s" % arguments.id)
        print("deleted.")
    return EXIT_OK


def print_reputation(answer):
    """The report as a table - one row per address and list - then the reasons, then
    what was deliberately not looked up, then when the last check was.

    The status column carries the word the API sends rather than a yes or a no,
    because "refused", "lookup_failed" and "list_not_answering" are not "not_listed",
    and a client that folded them together would report as clean a server no list has
    answered about.
    """

    rows = []
    for address in answer.get("addresses") or []:
        for result in address.get("results") or []:
            rows.append({
                "address": address.get("address", ""),
                "list": result.get("list", ""),
                "status": result.get("status", ""),
                "answer": result.get("code", ""),
                "listed since": when(result.get("listed_since")),
                "checked": when(result.get("checked")),
            })

    if rows:
        print_table(rows, ["address", "list", "status", "answer", "listed since", "checked"])
    else:
        print("(no outbound address is configured, so nothing is checked)")

    # A list's reason is a sentence, so it goes under the table rather than into a
    # column of it. A list that simply says no has nothing to explain.
    for address in answer.get("addresses") or []:
        for result in address.get("results") or []:
            if result.get("reason") and result.get("status") != "not_listed":
                print("")
                print("%s on %s: %s" % (address.get("address", ""), result.get("list", ""), result["reason"]))

    for skipped in answer.get("skipped") or []:
        print("")
        print("not looked up: %s (%s) %s" % (skipped.get("address", ""), skipped.get("source", ""),
                                             skipped.get("detail") or skipped.get("reason", "")))

    if answer.get("lists_problem"):
        print("")
        print("the lists: %s" % answer["lists_problem"])

    # The one fact that explains every other line when it is false. Without it a
    # table of "not_checked" reads as a server that is behind on its checks rather
    # than one that was never asked to make any.
    if not answer.get("enabled"):
        print("")
        print("the watch is off: nothing is checked and nothing will be, until")
        print("reputation_watch_enabled is turned on (hmctl settings set --set")
        print("reputation_watch_enabled=true, or the Deck's Settings page). An")
        print("upgraded server starts with it off.")

    print("")
    print("last checked: %s" % (when(answer.get("last_checked")) or "never"))

    run = answer.get("run")
    if isinstance(run, dict):
        print("this check: %s address(es), %s list(s), %s lookup(s)"
              % (run.get("addresses", 0), run.get("lists", 0), run.get("lookups", 0)))


def reputation_command(client, arguments):
    if arguments.verb == "check":
        answer = client.request("POST", "/api/v1/reputation/check", body={}) or {}
    else:
        answer = client.request("GET", "/api/v1/reputation") or {}

    if arguments.json:
        render(answer, True)
    else:
        print_reputation(answer)

    # Exit 1 when an address is listed, when a list could not be asked, and when a
    # pair has never been checked at all: a check nobody answered is not a clean
    # one, and neither is a check nobody has made. A monitoring job reads the exit
    # code rather than the table, so the code has to carry all three.
    #
    # not_checked is reachable for up to a day in ordinary use: an address added to
    # reputation_watch_addresses, or a zone added to reputation_watch_lists, has no
    # stored row until the next daily check, and the daily check is not brought
    # forward by the addition.
    #
    # With the watch OFF, not_checked is not a fault at all - it is every pair,
    # for ever, because nothing is being checked. An upgrade leaves the watch off
    # with the lists seeded, so a server with a delivery bind address would alarm
    # at every invocation with nothing in the table to explain it. A listing or an
    # unanswered list still exits 1 with the watch off, because those are answers
    # an earlier check got and they do not stop being true.
    counts = answer.get("counts") or {}
    if counts.get("listed") or counts.get("unknown"):
        return EXIT_ERROR

    if counts.get("not_checked") and answer.get("enabled"):
        return EXIT_ERROR

    return EXIT_OK


def profiles_command(client, arguments):
    """The setting profiles (docs/Profiles.md): the four, one's diff against this
    server, applying one - with --confirm TOKEN where an entry closes a listener,
    trusts an address range, turns a filter or a check off, or shortens how long
    data is kept, TOKEN being the diff's own, which ties the confirmation to the
    diff that was read - the way back, and keeping the shipped settings. The
    server decides everything; this prints what it said."""
    if arguments.verb == "profiles":
        answer = client.request("GET", "/api/v1/settings/profiles") or {}
        if arguments.json:
            render(answer, True)
            return EXIT_OK
        render(answer.get("profiles", []), False, ["name", "title", "entries", "changes", "confirmations", "skipped"])
        applied = answer.get("applied")
        if isinstance(applied, dict):
            print("")
            if applied.get("state") == "declined":
                print("the settings this version ships with were chosen; no profile is applied")
            else:
                print("last applied: %s (%s)" % (applied.get("profile"), applied.get("state")))
        snapshot = answer.get("snapshot")
        if isinstance(snapshot, dict) and snapshot.get("profile"):
            print("the way back from %s is held: hmctl settings revert-profile puts it back" % snapshot.get("profile"))
        elif isinstance(snapshot, dict):
            print("a way back is held and cannot be read: hmctl settings discard-profile discards it")
        return EXIT_OK

    if arguments.verb == "profile":
        answer = client.request("GET", "/api/v1/settings/profiles/%s" % urllib.parse.quote(arguments.name, safe="")) or {}
        if arguments.json:
            render(answer, True)
            return EXIT_OK
        print("%s - %s" % (answer.get("name"), answer.get("title")))
        rows = []
        for entry in answer.get("entries", []):
            note = []
            if entry.get("confirm"):
                note.append("asks for --confirm: " + str(entry.get("caution", "")).replace("_", " "))
            if entry.get("note"):
                note.append(entry.get("note"))
            if entry.get("environment"):
                note.append("set by the environment (%s)" % entry.get("environment"))
            if entry.get("refused"):
                note.append("refused: " + entry.get("refused"))
            rows.append({"store": entry.get("store"), "name": entry.get("name"), "now": entry.get("current"),
                         "profile": entry.get("value"), "changes": entry.get("changes"), "note": "; ".join(note)})
        render(rows, False, ["store", "name", "now", "profile", "changes", "note"])
        print("")
        print("%s would change, %s unchanged, %s skipped, %s refused" % (
            answer.get("changes"), answer.get("unchanged"), answer.get("skipped"), answer.get("refused")))
        if answer.get("blocked"):
            print(answer.get("blocked"))
        if answer.get("diff_token"):
            print("diff token: %s" % answer.get("diff_token"))
            if answer.get("confirmation_needed"):
                print("to apply it with the confirmation: hmctl settings apply-profile %s --confirm %s" % (answer.get("name"), answer.get("diff_token")))
        return EXIT_OK

    if arguments.verb == "apply-profile":
        # --confirm takes the diff's token, which hmctl settings profile NAME prints:
        # the confirmation is of that diff, and the server refuses it when the
        # settings moved since. --token ties an apply that needs no confirmation to
        # the diff the same way.
        body = {}
        if arguments.confirm:
            body = {"confirm": True, "diff_token": arguments.confirm}
        elif arguments.token:
            body = {"diff_token": arguments.token}
        try:
            answer = client.request("POST", "/api/v1/settings/profiles/%s/apply" % urllib.parse.quote(arguments.name, safe=""), body=body) or {}
        except ApiError as refused:
            detail = refused.body if isinstance(refused.body, dict) else {}
            if refused.status == 409 and detail.get("confirmation"):
                names = ", ".join(str(e.get("name")) for e in detail.get("confirmation_needed", []))
                raise ApiError(409, "%s Entries: %s. Read the diff with hmctl settings profile %s and run it again with --confirm and the diff's token." % (refused.message, names, arguments.name), refused.body)
            if refused.status == 409 and detail.get("stale"):
                raise ApiError(409, "%s Read it again with hmctl settings profile %s." % (refused.message, arguments.name), refused.body)
            raise
        if arguments.json:
            render(answer, True)
        else:
            print(answer.get("message", ""))
        return EXIT_OK if not answer.get("failed") else EXIT_ERROR

    if arguments.verb == "revert-profile":
        answer = client.request("POST", "/api/v1/settings/profiles/revert", body={}) or {}
        if arguments.json:
            render(answer, True)
        else:
            print(answer.get("message", ""))
            for entry in answer.get("entries", []):
                if entry.get("result") != "restored":
                    print("  %s: %s - %s" % (entry.get("name"), entry.get("result"), entry.get("reason") or ""))
        return EXIT_OK if not answer.get("failed") else EXIT_ERROR

    if arguments.verb == "decline-profile":
        answer = client.request("POST", "/api/v1/settings/profiles/decline", body={}) or {}
        render(answer, True) if arguments.json else print(answer.get("message", ""))
        return EXIT_OK

    answer = client.request("POST", "/api/v1/settings/profiles/discard", body={}) or {}
    render(answer, True) if arguments.json else print(answer.get("message", ""))
    return EXIT_OK


def settings_command(client, arguments):
    if arguments.verb in ("profiles", "profile", "apply-profile", "revert-profile", "discard-profile", "decline-profile"):
        return profiles_command(client, arguments)

    if arguments.verb == "breach-test":
        # The password breach check's endpoint, asked for the prefix 00000, which is
        # no password's; nothing is changed. Exit 1 when it could not be used, so a
        # monitoring job can alert on the exit code alone.
        body = {"url": arguments.endpoint} if arguments.endpoint else {}
        answer = client.request("POST", "/api/v1/settings/breach-check/test", body=body) or {}
        if arguments.json:
            render(answer, True)
        elif answer.get("reachable"):
            print("reachable: %s entries, %s, in %s ms" % (
                answer.get("entries"), "padded" if answer.get("padded") else "not padded", answer.get("milliseconds")))
        else:
            print("not reachable: %s" % answer.get("error", "no answer"))
        return EXIT_OK if answer.get("reachable") else EXIT_ERROR

    group = arguments.group or "settings"
    if group not in SETTING_GROUPS:
        raise ApiError(0, "unknown settings group %r; the groups are: %s" % (group, ", ".join(SETTING_GROUPS)))
    path = "/api/v1/settings" if group == "settings" else "/api/v1/settings/" + group

    if arguments.verb == "get":
        answer = client.request("GET", path)
        if arguments.name:
            if not isinstance(answer, dict) or arguments.name not in answer:
                raise ApiError(404, "no setting %r in the %s group" % (arguments.name, group))
            value = answer[arguments.name]
            print(json.dumps(value) if arguments.json else value)
            return EXIT_OK
        render(answer, arguments.json)
        return EXIT_OK

    body = body_from(arguments)
    if not body:
        raise ApiError(0, "nothing to set: give --set name=value")
    answer = client.request("PUT", path, body=body)
    render(answer if answer is not None else {"changed": sorted(body)}, arguments.json)
    return EXIT_OK


def ini_command(client, arguments):
    if arguments.verb == "list":
        answer = client.request("GET", "/api/v1/settings/ini")
        render(ini_rows(answer), arguments.json, ["name", "value"])
    elif arguments.verb == "get":
        render(client.request("GET", "/api/v1/settings/ini/%s" % arguments.name), arguments.json)
    elif arguments.verb == "set":
        answer = client.request("PUT", "/api/v1/settings/ini/%s" % arguments.name, body={"value": arguments.value})
        render(answer if answer is not None else {"name": arguments.name, "value": arguments.value}, arguments.json)
    elif arguments.verb == "delete":
        client.request("DELETE", "/api/v1/settings/ini/%s" % arguments.name)
        print("%s removed; the server's built-in default applies." % arguments.name)
    return EXIT_OK


def log_command(client, arguments):
    if arguments.verb == "list":
        render(collection(client.request("GET", "/api/v1/logs"), "logs"), arguments.json, ["name", "size", "modified"])
    else:
        answer = client.request("GET", "/api/v1/logs/%s" % arguments.name, query={"lines": arguments.lines})
        if isinstance(answer, dict) and "lines" in answer:
            for line in answer["lines"]:
                print(line)
        else:
            render(answer, arguments.json)
    return EXIT_OK


def read_backup_passphrase(arguments, confirm=False):
    """The backup passphrase, from a file or from the terminal.

    Never from the command line. Everything after the program name is in the
    process list, which every user of the machine can read and which several
    shells write to a history file as well - and this is the one secret that
    opens every mailbox on the server at once.

    None means none was offered, which is different from an empty one: an
    archive that is not encrypted needs none, and an empty passphrase is how the
    encryption is switched off.
    """
    if getattr(arguments, "passphrase_file", None):
        if arguments.passphrase_file == "-":
            line = sys.stdin.readline()
        else:
            with open(arguments.passphrase_file, encoding="utf-8") as handle:
                line = handle.readline()
        return line.rstrip("\r\n")

    if not getattr(arguments, "ask_passphrase", False):
        return None

    try:
        passphrase = getpass.getpass("Backup passphrase: ")
        if confirm and passphrase != getpass.getpass("The same passphrase again: "):
            raise ApiError(0, "the two passphrases are not the same; nothing has been changed")
        return passphrase
    except (EOFError, OSError):
        raise ApiError(0, "there is no terminal here to ask for the passphrase on; use --passphrase-file FILE instead")


def backup_restore_command(client, arguments):
    parts = {
        "restore_settings": bool(arguments.settings),
        "restore_domains": bool(arguments.domains),
        "restore_messages": bool(arguments.messages),
    }

    if not any(parts.values()):
        raise ApiError(0, "choose what to restore: --settings, --domains, --messages "
                          "(messages can only be restored together with --domains)")

    body = {"file": arguments.file}
    body.update(parts)

    passphrase = read_backup_passphrase(arguments)
    if passphrase is not None:
        body["passphrase"] = passphrase

    answer = client.request("POST", "/api/v1/backup/restore", body=body) or {}

    if arguments.json:
        render(answer, True)
        return EXIT_OK

    contains = answer.get("contains") or {}
    held = ", ".join(name for name in ("settings", "domains", "messages") if contains.get(name))
    print("restore started from %s%s; the archive holds %s." % (
        arguments.file, " (encrypted)" if answer.get("encrypted") else "", held or "nothing"))
    print("It runs in the server and replaces what it restores; `hmctl backup status` says how it went.")
    return EXIT_OK


def backup_passphrase_command(client, arguments):
    if arguments.action == "clear":
        client.request("PUT", "/api/v1/settings/backup", body={"encryption_passphrase": ""})
        print("the backup passphrase is cleared: backups from now on are written unencrypted.")
        return EXIT_OK

    passphrase = read_backup_passphrase(arguments, confirm=True)

    if passphrase is None:
        raise ApiError(0, "give the passphrase in a file with --passphrase-file FILE (or - for standard input), "
                          "or --ask to be asked for it here; it is never taken as an argument, where the process "
                          "list would show it to every user of this machine")

    if not passphrase:
        raise ApiError(0, "an empty passphrase would switch encryption off; say `hmctl backup passphrase clear` "
                          "if that is what you mean")

    client.request("PUT", "/api/v1/settings/backup", body={"encryption_passphrase": passphrase})
    print("the backup passphrase is set: backups from now on are encrypted with it, file names included.")
    print("Keep it somewhere other than beside the backups. A restore asks for it, and an archive whose "
          "passphrase is lost cannot be read by anybody - this server included.")
    return EXIT_OK


def mailbox_route(archive, address, suffix=""):
    """The mailbox routes name the archive and the account in the path; both
    are encoded, since an archive's name has spaces in it and an address an @."""
    return "/api/v1/backup/%s/mailboxes/%s%s" % (urllib.parse.quote(archive, safe=""), urllib.parse.quote(address, safe=""), suffix)


def backup_archives_command(client, arguments):
    answer = client.request("GET", "/api/v1/backup/archives") or {}
    if arguments.json:
        render(answer, True)
        return EXIT_OK
    archives = answer.get("archives") or []
    if not archives:
        print("no archives in the backup destination %s." % (answer.get("destination") or "(none set)"))
        return EXIT_OK
    print("archives in %s, newest first:" % answer.get("destination", ""))
    for archive in archives:
        print("  %s  %s  %s byte(s)%s" % (archive.get("created", ""), archive.get("name", ""), archive.get("size", 0),
                                         "  (encrypted)" if archive.get("encrypted") else ""))
    return EXIT_OK


def backup_mailbox_command(client, arguments):
    passphrase = read_backup_passphrase(arguments)
    headers = {"X-hMailServer-Backup-Passphrase": passphrase} if passphrase else None
    path = mailbox_route(arguments.archive, arguments.address)
    if arguments.folder:
        path += "?folder=" + urllib.parse.quote(arguments.folder, safe="")
    answer = client.request("GET", path, headers=headers) or {}
    if arguments.json:
        render(answer, True)
        return EXIT_OK
    print("%s in %s: %s message(s) in %s folder(s)%s" % (
        answer.get("address", arguments.address), answer.get("archive", arguments.archive), answer.get("messages", 0),
        len(answer.get("folders") or []), "" if answer.get("subjects") else "; " + (answer.get("subjects_note") or "subjects not read")))
    for folder in answer.get("folders") or []:
        print("  %s (%d)" % (folder.get("path", ""), len(folder.get("messages") or [])))
        for message in folder.get("messages") or []:
            print("    %-8s %s  %s  %s  %s byte(s)" % (message.get("id", ""), message.get("date", ""), message.get("from", ""),
                                                        message.get("subject", ""), message.get("size", 0)))
    return EXIT_OK


def backup_restore_mailbox_command(client, arguments):
    if arguments.merge and arguments.replace:
        raise ApiError(0, "choose one of --merge and --replace")
    if arguments.folder and not arguments.message:
        raise ApiError(0, "--folder names where one message goes; give --message ID with it")
    if arguments.message and (arguments.merge or arguments.replace):
        raise ApiError(0, "a mode does not apply to a single message: --message puts that one message back as merge "
                          "of it; leave --merge and --replace out")

    body = {}
    if arguments.message:
        body["message"] = arguments.message
        if arguments.folder:
            body["folder"] = arguments.folder
    else:
        body["mode"] = "replace" if arguments.replace else "merge"

    passphrase = read_backup_passphrase(arguments)
    if passphrase is not None:
        body["passphrase"] = passphrase

    answer = client.request("POST", mailbox_route(arguments.archive, arguments.address, "/restore"), body=body) or {}

    if arguments.json:
        render(answer, True)
        return EXIT_OK

    if arguments.message:
        print("restore of message %s of %s from %s started as job %s%s." % (
            arguments.message, arguments.address, arguments.archive, answer.get("job"),
            " into " + arguments.folder if arguments.folder else ""))
    else:
        print("%s of the mailbox of %s from %s started as job %s." % (
            "replace" if arguments.replace else "merge", arguments.address, arguments.archive, answer.get("job")))
    print("It runs in the server; `hmctl backup restore-status %s` says how it went." % answer.get("job"))
    return EXIT_OK


def backup_restore_status_command(client, arguments):
    answer = client.request("GET", "/api/v1/backup/restore-mailbox/%d" % arguments.job) or {}
    if arguments.json:
        render(answer, True)
        return EXIT_OK
    print("job %s is %s: %s of %s from %s; %s folder(s), %s added, %s skipped as still in the mailbox, %s failed%s" % (
        answer.get("job"), answer.get("state"), answer.get("mode"), answer.get("address"), answer.get("archive"),
        answer.get("folders", 0), answer.get("added", 0), answer.get("skipped", 0), answer.get("failed", 0),
        ("; " + answer["error"]) if answer.get("error") else "."))
    return EXIT_OK


def backup_command(client, arguments):
    if arguments.verb == "start":
        render(client.request("POST", "/api/v1/backup", body={}), arguments.json)
    elif arguments.verb == "restore":
        return backup_restore_command(client, arguments)
    elif arguments.verb == "passphrase":
        return backup_passphrase_command(client, arguments)
    elif arguments.verb == "archives":
        return backup_archives_command(client, arguments)
    elif arguments.verb == "mailbox":
        return backup_mailbox_command(client, arguments)
    elif arguments.verb == "restore-mailbox":
        return backup_restore_mailbox_command(client, arguments)
    elif arguments.verb == "restore-status":
        return backup_restore_status_command(client, arguments)
    else:
        render(client.request("GET", "/api/v1/backup"), arguments.json)
    return EXIT_OK


def rule_match_command(client, arguments):
    body = {"match_value": arguments.value, "match_type": arguments.type, "test_value": arguments.against}
    answer = client.request("POST", "/api/v1/rules/match", body=body)
    if arguments.json:
        render(answer, True)
    else:
        matched = answer.get("match") if isinstance(answer, dict) else answer
        print("match" if matched else "no match")
    return EXIT_OK if (not isinstance(answer, dict) or answer.get("match")) else EXIT_OK


def apppassword_command(client, arguments):
    base = "/api/v1/accounts/%s/app-passwords" % urllib.parse.quote(arguments.address, safe="@")
    if arguments.verb == "list":
        render(collection(client.request("GET", base), "app_passwords"), arguments.json,
               ["id", "name", "created", "last_used"])
    elif arguments.verb == "create":
        answer = client.request("POST", base, body={"name": arguments.name})
        render(answer, arguments.json)
        if not arguments.json and isinstance(answer, dict) and answer.get("password"):
            print("")
            print("This is the only time the server will show it.")
    elif arguments.verb == "delete":
        client.request("DELETE", "%s/%s" % (base, arguments.id))
        print("revoked.")
    return EXIT_OK


def device_command(client, arguments):
    """The clients that have signed in to an account, and the one call that ends
    what one of them holds. The list is the account holder's own (GET
    /api/v1/me/devices) read under the account's address by an administrator;
    revoking ends the client's browser session and drops its live IMAP, POP3 and
    SMTP connections, and with --credential deactivates the app password it
    signed in with, which is what stops a stolen phone signing in again."""
    base = "/api/v1/accounts/%s/devices" % urllib.parse.quote(arguments.address, safe="@")
    if arguments.verb == "list":
        render(collection(client.request("GET", base), "devices"), arguments.json,
               ["id", "protocol", "address", "agent", "credential", "credential_name", "last_seen", "sign_ins", "revoked"])
    elif arguments.verb == "revoke":
        query = []
        if getattr(arguments, "credential", False):
            query.append("credential=revoke")
        if getattr(arguments, "forget", False):
            query.append("forget=true")
        path = "%s/%s" % (base, arguments.id)
        if query:
            path += "?" + "&".join(query)
        answer = client.request("DELETE", path)
        render(answer, arguments.json)
        if not arguments.json and isinstance(answer, dict):
            print("%s session(s) ended, %s connection(s) dropped%s." % (
                answer.get("sessions_ended", 0), answer.get("connections_dropped", 0),
                ", the app password it used turned off" if answer.get("credential_disabled") else ""))
    return EXIT_OK


def member_command(client, arguments):
    base = "/api/v1/groups/%s/members" % arguments.group
    if arguments.verb == "list":
        render(collection(client.request("GET", base), "members"), arguments.json, ["id", "account_id", "address"])
    elif arguments.verb == "add":
        key = "address" if "@" in arguments.account else "account_id"
        value = arguments.account if key == "address" else int(arguments.account)
        render(client.request("POST", base, body={key: value}), arguments.json)
    elif arguments.verb == "remove":
        client.request("DELETE", "%s/%s" % (base, arguments.account))
        print("removed from the group.")
    return EXIT_OK


# ---------------------------------------------------------------------------
# DKIM key rotation
# ---------------------------------------------------------------------------

def when(seconds):
    """A time the rotation answers in seconds since the epoch, in UTC; 0 is none."""
    import datetime
    if not isinstance(seconds, (int, float)) or seconds <= 0:
        return ""
    moment = datetime.datetime.fromtimestamp(int(seconds), datetime.timezone.utc)
    return moment.strftime("%Y-%m-%d %H:%M UTC")


def print_rotation(status):
    """The status document as a person reads it: the phase, one block per key
    with the records to publish, keep or remove, then what is waiting. The
    record lines are ACTION HOST TXT VALUE, one per line, so that grep and awk
    find them."""
    if not isinstance(status, dict):
        render(status, False)
        return

    policy = status.get("policy") or {}
    print("%s: %s" % (status.get("domain", ""), status.get("phase", "")))
    if policy:
        print("schedule: %s, every %s day(s), propagation %s hour(s), retention %s day(s), %s-bit keys"
              % ("on" if policy.get("enabled") else "off", policy.get("interval_days"),
                 policy.get("propagation_hours"), policy.get("retention_days"), policy.get("key_bits")))

    slots = (
        ("current", (("since", "since"), ("next rotation due", "next_rotation_due"))),
        ("next", (("created", "created"), ("first seen", "first_seen"), ("switch due", "switch_due"))),
        ("retiring", (("since", "since"), ("retire due", "retire_due"))),
        ("retired", (("at", "at"),)),
    )
    for name, times in slots:
        slot = status.get(name)
        if not isinstance(slot, dict):
            continue
        facts = ["%s %s" % (label, when(slot.get(field))) for label, field in times if when(slot.get(field))]
        check = slot.get("check") if isinstance(slot.get("check"), dict) else None
        if check:
            facts.append("check: %s%s" % (check.get("status", ""),
                                           (" at " + when(check.get("checked"))) if when(check.get("checked")) else ""))
        print("")
        print("%-9s %s%s" % (name, slot.get("selector", ""), ("  (" + "; ".join(facts) + ")") if facts else ""))
        if slot.get("key_file"):
            print("          key file %s%s" % (slot["key_file"], "" if slot.get("generated") else " (not generated by the rotation)"))
        if slot.get("key_file_kept"):
            print("          key file kept: %s" % slot["key_file_kept"])
        if check and check.get("detail"):
            print("          found: %s" % check["detail"])
        for record in slot.get("records") or []:
            if isinstance(record, dict):
                line = "  %-8s %s %s %s" % (record.get("action", ""), record.get("host", ""),
                                           record.get("type", "TXT"), record.get("value", ""))
                print(line.rstrip())

    waiting = [w for w in (status.get("waiting") or []) if isinstance(w, dict)]
    if waiting:
        print("")
        for item in waiting:
            until = when(item.get("until"))
            print("waiting (%s): %s%s" % (item.get("step", ""), item.get("detail") or item.get("reason", ""),
                                          (" Until " + until + ".") if until else ""))

    error = status.get("last_error")
    if isinstance(error, dict) and error.get("text"):
        print("")
        print("last error%s: %s" % ((" at " + when(error.get("time"))) if when(error.get("time")) else "", error["text"]))


def dkim_command(client, arguments):
    if arguments.verb == "run":
        body = {}
        if arguments.seconds_ahead:
            body["seconds_ahead"] = arguments.seconds_ahead
        render(client.request("POST", "/api/v1/dkim/rotation/run", body=body), arguments.json)
        return EXIT_OK

    base = "/api/v1/domains/%s/dkim/rotation" % urllib.parse.quote(arguments.domain, safe="")
    try:
        if arguments.verb == "status":
            answer = client.request("GET", base)
        elif arguments.verb == "rotate":
            body = {"selector": arguments.selector} if arguments.selector else {}
            answer = client.request("POST", base + "/start", body=body)
        elif arguments.verb == "check":
            answer = client.request("POST", base + "/check", body={})
        elif arguments.verb == "switch":
            answer = client.request("POST", base + "/switch", body={"force": True} if arguments.force else {})
        elif arguments.verb == "retire":
            answer = client.request("POST", base + "/retire", body={})
        else:
            answer = client.request("DELETE", base + "/next")
    except ApiError as error:
        # A refusal carries the reason word and the status beside it: the status is
        # what shows what is in the way, so it is printed before the refusal.
        if error.status == 409 and isinstance(error.body, dict):
            if isinstance(error.body.get("status"), dict):
                if arguments.json:
                    render(error.body, True)
                else:
                    print_rotation(error.body["status"])
                    print("")
            reason = error.body.get("reason")
            raise ApiError(409, "%s%s" % (error.message, (" (%s)" % reason) if reason else ""), error.body)
        raise

    if arguments.json:
        render(answer, True)
    else:
        print_rotation(answer)
    return EXIT_OK


# ---------------------------------------------------------------------------
# Renaming a mailbox, and moving one to another domain


def print_mailbox_move(status):
    """What GET /api/v1/accounts/<address>/move answers, as sentences."""
    print("address       %s" % status.get("address", ""))
    if status.get("in_use"):
        print("in use        yes: %s" % status.get("in_use_reason", ""))
    else:
        print("in use        no")

    last = status.get("last_move")
    if not isinstance(last, dict):
        print("last move     never moved")
        return

    print("last move     %s -> %s" % (last.get("old_address", ""), last.get("new_address", "")))
    print("              %s, %s" % (last.get("stage_name", ""), last.get("finished") or last.get("started") or ""))
    if not last.get("complete"):
        # A row short of the last stage is a move that was interrupted, and the
        # server's application log says which half of the store to look at.
        print("              this move did not finish; see the server's application log")


def mailbox_command(client, arguments):
    base = "/api/v1/accounts/%s/move" % urllib.parse.quote(arguments.address, safe="@")

    if arguments.verb == "status":
        answer = client.request("GET", base)
        if arguments.json:
            render(answer, True)
        else:
            print_mailbox_move(answer)
        return EXIT_OK

    body = {"new_address": arguments.new_address, "keep_alias": not arguments.drop_alias}
    answer = client.request("POST", base, body=body)

    if arguments.json:
        render(answer, True)
        return EXIT_OK

    print("%s moved to %s." % (answer.get("old_address", arguments.address), answer.get("new_address", arguments.new_address)))
    print("  %s message(s) in %s folder(s)." % (answer.get("messages", 0), answer.get("folders", 0)))
    print("  %s alias(es) re-pointed, %s list membership(s), %s queued recipient(s), %s quarantined message(s)."
          % (answer.get("aliases_repointed", 0), answer.get("list_memberships", 0),
             answer.get("queued_recipients", 0), answer.get("quarantine_entries", 0)))
    if answer.get("alias_kept"):
        print("  The old address is kept as an alias.")
    else:
        print("  The old address is not kept: mail to it will be refused.")

    # A rewrite that failed writes nothing and so counts nothing: without this
    # line, "0 alias(es) re-pointed" reads as a move that had no aliases to
    # re-point rather than one whose aliases are still naming the old address.
    if answer.get("references_incomplete"):
        print("  Some references to the old address could not be rewritten. The server's application log says which.")
    return EXIT_OK


# ---------------------------------------------------------------------------
# Accounts from a spreadsheet
# ---------------------------------------------------------------------------

# The columns, in the order the export writes them. Each is a field of
# POST /api/v1/domains/<domain>/accounts under the create's own name, and the
# export reads the same names from the domain's listing, so a file exported and
# given a password column goes back in as it came out. The create refuses any
# other field by name, so a column not in this list is refused here, before a
# request is made - which is what lets a dry run say so as well.
CSV_FIELDS = ["address", "password", "first_name", "last_name", "active", "max_size_mb", "admin_level"]

# How a cell is sent. active is a boolean and max_size_mb a number; every other
# column is text, whatever it looks like - a password of digits alone, or a
# surname of "True", is a string. Those two, admin_level, a missing password and
# a name past its column are checked here in the server's own words, so a dry
# run refuses what the create would refuse. What only the server can judge - the
# password policy, the domain's limits - is refused when the row is sent.
CSV_TRUE = ("true", "yes", "on", "1")
CSV_FALSE = ("false", "no", "off", "0")
CSV_ADMIN_LEVELS = ("user", "domain", "server")

# The width of accountpersonfirstname and accountpersonlastname, counted as the
# Windows server counts it and as SQL Server's nvarchar(60) holds it: in UTF-16
# code units, so a character outside the Basic Multilingual Plane counts two.
# PowerShell's String.Length counts the same way, so the two clients refuse
# the same names.
CSV_NAME_WIDTH = 60


def utf16_length(text):
    return len(text.encode("utf-16-le")) // 2


def account_csv_columns(path, fieldnames, rows):
    """The header's columns, lower-cased and trimmed, or a refusal naming the
    ones the create does not take."""
    columns = [(name or "").strip().lower() for name in (fieldnames or [])]

    if "address" not in columns:
        raise ApiError(0, "%s has no 'address' column; its columns are: %s" %
                       (path, ", ".join(name for name in (fieldnames or []) if name)))

    unknown = []
    for original, column in zip(fieldnames or [], columns):
        if column in CSV_FIELDS or column in unknown:
            continue
        # A header cell left blank over cells left blank is a spreadsheet's
        # trailing comma, not a column.
        if column == "" and not any((row.get(original) or "").strip() for row in rows):
            continue
        unknown.append(column)

    if unknown:
        hint = ""
        if "name" in unknown:
            hint = " A person's name goes in two columns, first_name and last_name."
        raise ApiError(0, "%s has a column the server does not take: %s. The columns are %s.%s" %
                       (path, ", ".join(repr(column) for column in unknown), ", ".join(CSV_FIELDS), hint))

    return columns


def account_csv_body(row):
    """One row as the create's body, or (None, reason) when the create would
    refuse it. An empty cell is not sent, so the server's default applies."""
    if None in row:
        return None, "more cells than the header has columns (a comma inside a value needs the value in quotes)"

    body = {}
    for original, value in row.items():
        column = (original or "").strip().lower()
        if value is None or column == "":
            continue

        # Every cell is trimmed but the password's: a space at either end of a
        # spreadsheet cell is far more often a slip than part of a name, and
        # trimming one out of a password would make an account whose password
        # is not the one in the file, without a word. So a password with white
        # space at an edge is refused instead, and a dry run shows it. The two
        # clients send the same body for one file.
        text = value.strip()
        if text == "":
            continue

        if column == "password" and text != value:
            return None, ("the password begins or ends with white space, which a spreadsheet cell rarely means; "
                          "take it out of the cell, or set that password after the import")

        if column == "active":
            word = text.lower()
            if word in CSV_TRUE:
                body[column] = True
            elif word in CSV_FALSE:
                body[column] = False
            else:
                return None, "active must be true or false, not %r" % text
        elif column == "max_size_mb":
            if not (text.isascii() and text.isdigit()) or len(text) > 10 or int(text) > 2147483647:
                return None, "max_size_mb must be a whole number of megabytes, 0 for no limit, not %r" % text
            body[column] = int(text)
        elif column == "admin_level":
            if text.lower() not in CSV_ADMIN_LEVELS:
                return None, "admin_level must be user, domain or server, not %r" % text
            body[column] = text.lower()
        elif column in ("first_name", "last_name"):
            if utf16_length(text) > CSV_NAME_WIDTH:
                return None, ("first_name and last_name are at most 60 characters each; this %s has %d"
                              % (column, utf16_length(text)))
            body[column] = text
        else:
            body[column] = text

    if not body.get("password"):
        return None, "address and password are required; this row has no password"

    return body, None


def accounts_import(client, arguments):
    """A CSV of accounts into their domains, with a dry run that says what would happen.

    The header row names the columns, which are CSV_FIELDS in any order: address
    and password are required on every row, and the rest may be left out or left
    empty. A row whose address already exists is left alone and counted, because
    an import that silently overwrote a password would be a way to lock a hundred
    people out with one command.
    """
    try:
        with io.open(arguments.file, encoding="utf-8-sig", newline="") as handle:
            reader = csv.DictReader(handle)
            rows = list(reader)
            fieldnames = reader.fieldnames
    except UnicodeDecodeError:
        # A spreadsheet's plain "CSV" is often the ANSI code page. Guessing
        # which one would import a name wrong without a word.
        raise ApiError(0, "%s is not UTF-8. Save it as CSV UTF-8 and import it again." % arguments.file)

    if not rows:
        raise ApiError(0, "%s has no rows under its header" % arguments.file)

    account_csv_columns(arguments.file, fieldnames, rows)
    address_key = [name for name in fieldnames if (name or "").strip().lower() == "address"][0]

    # The domains to read the existing addresses from: the ones named, or the
    # ones the file's own addresses are in. Accounts are listed UNDER their
    # domain (/api/v1/domains/<domain>/accounts); asking /api/v1/accounts with a
    # query is not a route, and the 404 it answers would make every address in
    # the file look new.
    domains = [d.lower() for d in (arguments.domain or [])]
    if not domains:
        seen = set()
        for row in rows:
            address = (row.get(address_key) or "").strip()
            if "@" in address:
                seen.add(address.split("@", 1)[1].lower())
        domains = sorted(seen)

    existing = set()
    for domain in domains:
        try:
            answer = client.request("GET", "/api/v1/domains/%s/accounts" % urllib.parse.quote(domain, safe=""))
        except ApiError as error:
            # A domain in the file that the server does not have: every address
            # in it is new, which is what an empty answer means here. The create
            # will fail with the server's own words if the domain is missing.
            if error.status == 404:
                continue
            raise
        for account in collection(answer, "accounts"):
            existing.add(str(account.get("address", "")).lower())

    made, skipped, failed = [], [], []
    for number, row in enumerate(rows, start=2):
        address = (row.get(address_key) or "").strip()
        if not address:
            failed.append((number, "(blank)", "no address"))
            continue
        if address.lower() in existing:
            skipped.append(address)
            continue
        if "@" not in address:
            failed.append((number, address, "not an e-mail address"))
            continue

        body, reason = account_csv_body(row)
        if body is None:
            failed.append((number, address, reason))
            continue

        if arguments.dry_run:
            made.append(address)
            continue

        try:
            client.request("POST", "/api/v1/domains/%s/accounts"
                           % urllib.parse.quote(address.split("@", 1)[1], safe=""), body=body)
            made.append(address)
        except ApiError as error:
            failed.append((number, address, error.message))

    if arguments.json:
        render({"made": made, "skipped": skipped,
                "failed": [{"row": r, "address": a, "error": e} for r, a, e in failed],
                "dry_run": bool(arguments.dry_run)}, True)
    else:
        verb = "would be made" if arguments.dry_run else "made"
        print("%d %s, %d already there, %d refused." % (len(made), verb, len(skipped), len(failed)))
        for address in made:
            print("  + %s" % address)
        for address in skipped:
            print("  = %s (exists; left alone)" % address)
        for number, address, message in failed:
            print("  ! line %d %s: %s" % (number, address, message))

    return EXIT_ERROR if failed else EXIT_OK


def accounts_export(client, arguments):
    """A domain's accounts as the import reads them, less the password column:
    the server does not have the passwords to give."""
    answer = client.request("GET", "/api/v1/domains/%s/accounts"
                            % urllib.parse.quote(arguments.domain, safe=""))
    accounts = collection(answer, "accounts")

    fields = [f for f in CSV_FIELDS if f != "password"]
    handle = sys.stdout if arguments.file == "-" else io.open(arguments.file, "w", encoding="utf-8", newline="")
    try:
        writer = csv.DictWriter(handle, fieldnames=fields, extrasaction="ignore")
        writer.writeheader()
        for account in accounts:
            cells = {}
            for field in fields:
                value = account.get(field, "")
                # The JSON words, so the file reads the same whichever client wrote it.
                cells[field] = ("true" if value else "false") if isinstance(value, bool) else value
            writer.writerow(cells)
    finally:
        if handle is not sys.stdout:
            handle.close()
            print("%d accounts written to %s." % (len(accounts), arguments.file))
    return EXIT_OK


# ---------------------------------------------------------------------------
# The whole configuration as one document
# ---------------------------------------------------------------------------

# What a configuration document holds, in the order it is applied: a domain
# before the accounts in it, a rule after the accounts it names. Each entry is
# the resource's key in RESOURCES, the field that identifies one, and whether it
# is listed under a parent.
CONFIG_SECTIONS = [
    {"name": "settings", "kind": "settings"},
    {"name": "ini", "kind": "ini"},
    {"name": "domains", "kind": "resource", "resource": "domain"},
    {"name": "accounts", "kind": "resource", "resource": "account"},
    {"name": "aliases", "kind": "resource", "resource": "alias"},
    {"name": "lists", "kind": "resource", "resource": "list"},
    {"name": "spam_traps", "kind": "resource", "resource": "spamtrap"},
    {"name": "groups", "kind": "resource", "resource": "group"},
    {"name": "rules", "kind": "resource", "resource": "rule"},
    {"name": "routes", "kind": "resource", "resource": "route"},
    {"name": "ports", "kind": "resource", "resource": "port"},
    {"name": "ipranges", "kind": "resource", "resource": "iprange"},
]

# Fields the server answers with that describe the world rather than the
# configuration, and that must not travel into a document or be compared: an
# identifier the server allocates, a count it computes, a time it recorded, or a
# piece of another field it splits out for a reader (a spam trap's local_part,
# which is its address up to the @). A field here that reached a create would be
# refused - the write routes take the fields they own and nothing else - so the
# document would not apply to the server it was exported from.
NOT_CONFIGURATION = {
    "id", "account_id", "domain_id", "rule_id", "route_id", "parent_id",
    "account_count", "alias_count", "member_count", "size_mb", "size",
    "created", "modified", "last_used", "last_login", "last_logon",
    "local_part",
}

# A domain's selector and key files while its DKIM key rotation is on. The rotation
# moves them on its own schedule, so they are the server's state and not the
# document's: an export taken before a switch would otherwise differ from the
# server after it, and applying it would ask for the old key back - which the
# server refuses with 409. Left out of the document and out of the comparison,
# and not sent, whenever the domain (on the server or in the document) has the
# rotation on.
#
# The server holds them for a while with the schedule off too: while a rotation
# started by hand has a next key staged or an old selector still being kept, it
# refuses a change to them just the same. The document keeps them then - they are
# the domain's configuration once that rotation is over - but a comparison that
# finds them different asks the server (rotation_in_flight), and an update leaves
# them out while the server says it holds them.
ROTATION_OWNED = ("dkim_selector", "dkim_private_key_file", "dkim_secondary_selector", "dkim_secondary_private_key_file")

CONFIG_VERSION = 1

# The live update's trust: the five [Settings] keys a server from 6.3.6 reads
# from its own hMailServer.ini alone and refuses to change over the REST API
# (docs/UpdateFeed.md, "Where the trust is read from"). They say what that
# machine's updates are checked against, which is the machine's and not the
# configuration's, so they are left out of the document as the keys the
# server's environment sets are. A document an earlier hmctl exported can
# carry them: diff and apply leave them out and say so, rather than plan a
# write the server refuses and fail the whole apply for it.
TRUST_ANCHOR_KEYS = ("UpdateSigningIdentity", "UpdateSigningIssuer", "UpdateSourceRepository",
                     "UpdateTrustRootsFile", "UpdateLogPublicKeyFile")


def is_trust_anchor(name):
    """Whether an ini key is one of the live update's trust, matched without case
    as the server matches it."""
    return str(name).lower() in set(key.lower() for key in TRUST_ANCHOR_KEYS)


def trust_anchors_in(document):
    """The trust keys a document's ini section carries, which diff and apply
    leave out: an export from before they were left out of it."""
    ini = (document.get("sections") or {}).get("ini") or {}
    return sorted(key for key in ini if is_trust_anchor(key))


def say_trust_anchors_skipped(document, as_json):
    """One line naming the trust keys a document carries and diff or apply left
    out, on the standard error under --json so that the JSON stays JSON."""
    skipped = trust_anchors_in(document)
    if not skipped:
        return
    line = ("Skipped %s: the live update's trust, which a server reads from its own hMailServer.ini "
            "and does not take over the REST API (docs/UpdateFeed.md, \"Where the trust is read from\"). "
            "Set it in that server's file. An export from this hmctl leaves it out." % ", ".join(skipped))
    print(line, file=sys.stderr if as_json else sys.stdout)


def without_rotation_owned(row, rotation_on=None):
    if rotation_on is None:
        rotation_on = row.get("dkim_rotation_enabled") is True
    if not rotation_on:
        return row
    return dict((k, v) for k, v in row.items() if k not in ROTATION_OWNED)


def rotation_in_flight(client, domain):
    """Whether the server's DKIM key rotation holds a domain's selector and key
    files right now - its schedule on, a next key staged, or an old selector still
    being kept - which is when the server refuses a change to them with 409. A
    server without the route, or a key that may not read it, answers no, and the
    update is sent as before."""
    try:
        answer = client.request("GET", "/api/v1/domains/" + urllib.parse.quote(domain, safe="") + "/dkim/rotation")
    except ApiError as error:
        if error.status in (403, 404):
            return False
        raise
    if not isinstance(answer, dict):
        return False
    policy = answer.get("policy") if isinstance(answer.get("policy"), dict) else {}
    return policy.get("enabled") is True or answer.get("next") is not None or answer.get("retiring") is not None


def configuration_key(section, row):
    """The field that identifies one entry of a section, as a person names it."""
    if section["kind"] != "resource":
        return None
    resource = RESOURCES[section["resource"]]
    identifier = resource["id"]
    if identifier in row:
        return str(row[identifier])
    # An alias is identified by the address it answers for, which the listing
    # calls "name"; a resource whose id field is not in the row falls back to
    # whatever unique text it does carry.
    for candidate in ("address", "name", "value"):
        if candidate in row:
            return str(row[candidate])
    return None


def strip_configuration(row):
    # <key>_levels is what the server says about a key - how deep it may be set
    # (docs/SettingLevels.md) - not configuration: a document carrying it would
    # have it applied back, and the server refuses it.
    return dict((k, v) for k, v in row.items()
                if k not in NOT_CONFIGURATION and not str(k).endswith("_id") and not str(k).endswith("_levels"))


def strip_environment(row):
    # <key>_environment says the server's environment sets <key>
    # (docs/EnvironmentConfiguration.md). Neither is the configuration's: the
    # value is the deployment's, stored again at every start, and a value a file
    # supplied is answered as (set) or null - applied back, either would be
    # refused (409, 400). Both are left out of the document.
    managed = set(str(k)[:-len("_environment")] for k in row if str(k).endswith("_environment"))
    return dict((k, v) for k, v in row.items() if k not in managed and not str(k).endswith("_environment"))


def read_configuration(client, domains=None):
    """The live configuration as a document. Deterministic: sorted throughout,
    no time of day in it, nothing the server allocates - so two exports of an
    unchanged server are the same bytes and a diff means a change."""
    document = {"version": CONFIG_VERSION, "sections": {}}

    for section in CONFIG_SECTIONS:
        if section["kind"] == "settings":
            groups = {}
            for group in SETTING_GROUPS:
                if group == "directories":
                    continue          # paths of this machine, not configuration to copy
                path = "/api/v1/settings" if group == "settings" else "/api/v1/settings/" + group
                try:
                    answer = client.request("GET", path)
                except ApiError as error:
                    if error.status in (403, 404):
                        continue
                    raise
                if isinstance(answer, dict):
                    groups[group] = dict(sorted(strip_configuration(strip_environment(answer)).items()))
            document["sections"]["settings"] = groups
            continue

        if section["kind"] == "ini":
            try:
                answer = client.request("GET", "/api/v1/settings/ini")
            except ApiError as error:
                if error.status in (403, 404):
                    continue
                raise
            # The keys the server's environment sets, whatever it sets them from:
            # the route's environment map, {key: "variable" | "file"}. As in the
            # groups (strip_environment), such a value is the deployment's, stored
            # again at every start, and applied to a server whose environment sets
            # it the PUT would be refused (409) - so it is not in the document.
            environment = answer.get("environment") if isinstance(answer, dict) else None
            managed = set(str(k).lower() for k in (environment or {}))
            values = {}
            for row in ini_rows(answer):
                if "name" not in row or "value" not in row:
                    continue
                # A secret is listed as "(set)", never as its value. It stays out of
                # the document: a document is for a repository, and one that carried
                # the placeholder would write "(set)" over the real secret on apply.
                if row.get("secret"):
                    continue
                if str(row["name"]).lower() in managed:
                    continue
                # The live update's trust, which is the machine's: see
                # TRUST_ANCHOR_KEYS.
                if is_trust_anchor(row["name"]):
                    continue
                values[str(row["name"])] = row.get("value")
            document["sections"]["ini"] = dict(sorted(values.items()))
            continue

        resource = RESOURCES[section["resource"]]
        rows = []
        if resource.get("parent") == "domain":
            for domain in (domains or []):
                path = resource["collection"].replace("{parent}", urllib.parse.quote(domain, safe=""))
                try:
                    answer = client.request("GET", path)
                except ApiError as error:
                    if error.status == 404:
                        continue
                    raise
                for row in collection(answer, resource["key"]):
                    if isinstance(row, dict):
                        entry = strip_configuration(row)
                        entry["domain"] = domain
                        rows.append(entry)
        elif resource.get("parent"):
            continue                  # a resource under an account: not in the document yet
        else:
            try:
                answer = client.request("GET", resource["collection"])
            except ApiError as error:
                # A server older than this file does not carry every resource,
                # and a read-only key may be refused one. Either way the
                # document says nothing about that section rather than the
                # export failing whole.
                if error.status in (403, 404):
                    continue
                raise
            for row in collection(answer, resource["key"]):
                if isinstance(row, dict):
                    entry = strip_configuration(row)
                    if section["name"] == "domains":
                        entry = without_rotation_owned(entry)
                    rows.append(entry)

        rows.sort(key=lambda r: json.dumps(r, sort_keys=True))
        document["sections"][section["name"]] = rows

        if section["name"] == "domains":
            domains = sorted(str(r.get("name")) for r in rows if r.get("name"))

    return document


def configuration_plan(live, wanted, in_flight=None):
    """What would have to happen to make the server match the document: a list
    of (section, verb, key, detail). Nothing is decided here about whether it
    may happen - that is apply's business. in_flight, when given, answers for a
    domain name whether the server's key rotation holds its selectors; it is
    asked only about a domain whose selectors differ and whose schedule is off."""
    plan = []

    live_sections = live.get("sections", {})
    wanted_sections = wanted.get("sections", {})

    for name in ("settings",):
        for group in sorted(set(list(live_sections.get(name, {}).keys()) + list(wanted_sections.get(name, {}).keys()))):
            here = (live_sections.get(name) or {}).get(group, {})
            there = (wanted_sections.get(name) or {}).get(group, {})
            for key in sorted(set(list(here.keys()) + list(there.keys()))):
                if key not in there:
                    continue          # the document says nothing about it: leave it alone
                if here.get(key) != there.get(key):
                    plan.append((name + "/" + group, "set", key, "%r -> %r" % (here.get(key), there.get(key))))

    here = live_sections.get("ini", {})
    there = wanted_sections.get("ini", {})
    for key in sorted(there.keys()):
        # Left out, and said by the caller (say_trust_anchors_skipped): the
        # server takes none of them over the REST API.
        if is_trust_anchor(key):
            continue
        if here.get(key) != there.get(key):
            plan.append(("ini", "set", key, "%r -> %r" % (here.get(key), there.get(key))))

    for section in CONFIG_SECTIONS:
        if section["kind"] != "resource":
            continue
        name = section["name"]
        here_rows = {}
        for row in live_sections.get(name, []) or []:
            key = configuration_key(section, row)
            if key:
                here_rows[key] = row
        there_rows = {}
        for row in wanted_sections.get(name, []) or []:
            key = configuration_key(section, row)
            if key:
                there_rows[key] = row

        if name == "domains":
            for key in there_rows:
                here = here_rows.get(key)
                there = there_rows[key]
                owned = (here or {}).get("dkim_rotation_enabled") is True or there.get("dkim_rotation_enabled") is True
                if (not owned and here is not None and in_flight is not None
                        and any(here.get(k) != there.get(k) for k in ROTATION_OWNED)):
                    owned = in_flight(key)
                if owned:
                    there_rows[key] = without_rotation_owned(there, True)
                    if here is not None:
                        here_rows[key] = without_rotation_owned(here, True)

        for key in sorted(there_rows):
            if key not in here_rows:
                plan.append((name, "create", key, json.dumps(there_rows[key], sort_keys=True)))
            elif here_rows[key] != there_rows[key]:
                changed = sorted(k for k in set(list(here_rows[key].keys()) + list(there_rows[key].keys()))
                                 if here_rows[key].get(k) != there_rows[key].get(k))
                plan.append((name, "update", key, ", ".join(changed)))

        for key in sorted(here_rows):
            if key not in there_rows:
                plan.append((name, "delete", key, ""))

    return plan


def render_plan(plan, as_json):
    if as_json:
        render([{"section": s, "verb": v, "key": k, "detail": d} for s, v, k, d in plan], True)
        return

    if not plan:
        print("The server already matches the document.")
        return

    counts = {}
    for section, verb, key, detail in plan:
        counts[verb] = counts.get(verb, 0) + 1
        print("  %-8s %-10s %s%s" % (verb, section, key, ("  (" + detail + ")") if detail else ""))
    print("")
    print(", ".join("%d to %s" % (counts[v], v) for v in sorted(counts)))


def config_export(client, arguments):
    document = read_configuration(client)
    text = json.dumps(document, indent=2, sort_keys=True)
    if not arguments.file or arguments.file == "-":
        print(text)
    else:
        with io.open(arguments.file, "w", encoding="utf-8") as handle:
            handle.write(text + "\n")
        print("Written to %s." % arguments.file)
    return EXIT_OK


def config_changed(client, arguments):
    """Every setting whose value is not the one this version ships with.

    One route answers the whole of it and this prints what the server said: the
    defaults are the server's own, read from the source it was built from, so
    this file holds no copy of them to go stale. The same document the Control
    Deck's page and COM's Settings.GetChangedFromDefault show, which is the
    point - three tools that describe the same server differently are worse
    than two that say nothing.

    A difference is NOT an exit code here, which is the one place this parts
    company with `config diff`. Every real server has changed something, so a
    non-zero exit would be the normal answer and every script would learn to
    ignore it.

    The route answers to the administrator password and to nothing else, as
    `ini list` does: an api key is refused and so is an account holding a role,
    because the settings include the server's secrets and the commands it runs.
    """
    answer = client.request("GET", "/api/v1/settings/changed")
    if arguments.json:
        render(answer, True)
        return EXIT_OK

    rows = answer.get("settings", []) if isinstance(answer, dict) else []
    shown = []
    for row in rows:
        if not isinstance(row, dict):
            continue
        entry = dict(row)
        # A name the server ships no default for: the column would otherwise be
        # empty, which reads as "the default is empty" rather than "there is none".
        if entry.get("known") is False:
            entry["default"] = "(none: this server does not read it)"
        # A value the file's [SettingsOverride] section is forcing over whatever is
        # stored. Said beside the value rather than left out, because it is the one
        # thing here that no page and no configuration export shows, and reading
        # "DnsServer 10.0.0.1" while the stored value is something else is how an
        # afternoon gets lost.
        if entry.get("overridden"):
            entry["value"] = "%s (forced from [SettingsOverride])" % entry.get("value", "")
        # A value the server's environment sets (docs/EnvironmentConfiguration.md):
        # changed from stock by the deployment, stored again at every start, and
        # refused to every interface - so said beside it, with where it came from.
        if entry.get("environment") in ("variable", "file"):
            entry["value"] = "%s (set by the environment%s)" % (
                entry.get("value", ""), ", from a file" if entry.get("environment") == "file" else "")
        entry.pop("known", None)
        entry.pop("secret", None)
        entry.pop("overridden", None)
        entry.pop("environment", None)
        shown.append(entry)

    render(shown, False, ["store", "name", "value", "default"])

    if isinstance(answer, dict):
        print("%d of %d settings are not what this version ships with." %
              (len(rows), int(answer.get("considered") or 0)))
    return EXIT_OK


def config_diff(client, arguments):
    with io.open(arguments.file, encoding="utf-8") as handle:
        wanted = json.load(handle)
    live = read_configuration(client)
    plan = configuration_plan(live, wanted, lambda name: rotation_in_flight(client, name))
    render_plan(plan, arguments.json)
    say_trust_anchors_skipped(wanted, arguments.json)
    # A difference is an exit code, so that a pipeline can gate on it the way it
    # gates on a failing test. A trust key the document carries is not one: it
    # is left out, and the line above says so.
    return EXIT_OK if not plan else EXIT_ERROR


def config_apply(client, arguments):
    with io.open(arguments.file, encoding="utf-8") as handle:
        wanted = json.load(handle)
    live = read_configuration(client)
    plan = configuration_plan(live, wanted, lambda name: rotation_in_flight(client, name))

    deletions = [step for step in plan if step[1] == "delete"]
    if deletions and not arguments.allow_delete:
        plan = [step for step in plan if step[1] != "delete"]

    if not arguments.force:
        render_plan(plan, arguments.json)
        say_trust_anchors_skipped(wanted, arguments.json)
        if deletions and not arguments.allow_delete:
            print("")
            print("%d entr(ies) on the server are not in the document. They are left alone; "
                  "--allow-delete removes them." % len(deletions))
        if plan:
            print("")
            print("Nothing was changed. Add --force to apply this plan.")
        return EXIT_OK

    say_trust_anchors_skipped(wanted, False)

    applied, refused = 0, []
    for section, verb, key, detail in plan:
        try:
            apply_one(client, wanted, section, verb, key)
            applied += 1
            print("  %-8s %-10s %s" % (verb, section, key))
        except ApiError as error:
            refused.append((section, verb, key, error.message))
            print("  ! %-6s %-10s %s: %s" % (verb, section, key, error.message))

    print("")
    print("%d applied, %d refused." % (applied, len(refused)))
    if deletions and not arguments.allow_delete:
        print("%d entr(ies) not in the document were left alone; --allow-delete removes them." % len(deletions))
    return EXIT_ERROR if refused else EXIT_OK


def apply_one(client, wanted, section, verb, key):
    """One step of a plan. Kept small and obvious: the interesting decisions are
    all in configuration_plan, and this only speaks HTTP."""
    sections = wanted.get("sections", {})

    if section.startswith("settings/"):
        group = section.split("/", 1)[1]
        path = "/api/v1/settings" if group == "settings" else "/api/v1/settings/" + group
        client.request("PUT", path, body={key: sections["settings"][group][key]})
        return

    if section == "ini":
        client.request("PUT", "/api/v1/settings/ini/" + urllib.parse.quote(key, safe=""),
                       body={"value": sections["ini"][key]})
        return

    definition = None
    for candidate in CONFIG_SECTIONS:
        if candidate["name"] == section:
            definition = candidate
            break
    if definition is None:
        raise ApiError(0, "no such section %r" % section)

    resource = RESOURCES[definition["resource"]]
    row = None
    for entry in sections.get(section, []) or []:
        if configuration_key(definition, entry) == key:
            row = dict(entry)
            break

    parent = row.pop("domain", None) if row else None

    if verb == "delete":
        path = resource["item"].replace("{id}", urllib.parse.quote(key, safe="@"))
        if "{parent}" in path:
            raise ApiError(0, "a %s is deleted under its parent, which this plan does not carry" % section)
        client.request("DELETE", path)
        return

    if row is None:
        raise ApiError(0, "the document has no %s named %s" % (section, key))

    if section == "domains" and verb == "update":
        # Asked of the server at the moment of the update, not of the document: the
        # schedule may be on there and off here, or a rotation started by hand may
        # be holding the selectors with the schedule off on both.
        row = without_rotation_owned(row, row.get("dkim_rotation_enabled") is True or rotation_in_flight(client, key))

    if verb == "create":
        path = resource["collection"]
        if "{parent}" in path:
            if not parent:
                raise ApiError(0, "the document's %s entry %s carries no domain" % (section, key))
            path = path.replace("{parent}", urllib.parse.quote(parent, safe=""))
        if definition["resource"] == "account" and "password" not in row:
            raise ApiError(0, "an account is not created without a password; add one to the document "
                              "for %s, or make the account first" % key)
        client.request("POST", path, body=row)
        return

    # An update. Some resources have none - a spam trap is an address and nothing
    # else, so there is nothing about one to change and no PUT route to send it
    # to. Said here rather than discovered as a 404, since a plan that asks for
    # one means the two sides disagree about a field that is not configuration.
    if "update" not in resource.get("verbs", ()):
        raise ApiError(0, "a %s is created or removed, never changed; the document and the server differ "
                          "about %s in a way nothing can apply" % (section, key))

    path = resource["item"].replace("{id}", urllib.parse.quote(key, safe="@"))
    if "{parent}" in path:
        if not parent:
            raise ApiError(0, "the document's %s entry %s carries no domain" % (section, key))
        path = path.replace("{parent}", urllib.parse.quote(parent, safe=""))
    client.request("PUT", path, body=row)


def traphits_command(client, arguments):
    """What has addressed a spam trap, and the clearing of that record."""
    if arguments.verb == "clear":
        render(client.request("DELETE", "/api/v1/spam-traps/hits"), arguments.json)
        return EXIT_OK

    if arguments.domain:
        path = "/api/v1/domains/%s/spam-traps/hits" % urllib.parse.quote(arguments.domain, safe="")
    else:
        path = "/api/v1/spam-traps/hits"

    query = {}
    if arguments.limit:
        query["limit"] = arguments.limit

    answer = client.request("GET", path, query=query or None)
    render(collection(answer, "items"), arguments.json,
           ["time", "address", "ip_address", "sender", "banned", "blocked", "note"])
    return EXIT_OK


def sample_domain_command(client, arguments):
    """The throwaway example.test, made and removed in one command.

    Three verbs and no options. There is exactly one sample domain, it is always
    example.test, and what is in it is fixed - a sample with options is a
    configuration screen, which is the thing this exists to let somebody skip.

    Exit 2 for a refusal rather than 1: "it is already there", "there is none"
    and "that example.test is not mine" are answers, not failures, and a script
    that runs create at the start of a demonstration should be able to tell them
    from a server that could not be reached. The server says all three with 409,
    which main() would otherwise map to EXIT_ERROR like everything else, so the
    three calls are made here rather than left to it.
    """
    try:
        if arguments.verb == "create":
            answer = client.request("POST", "/api/v1/setup/sample-domain", body={}) or {}
        elif arguments.verb == "remove":
            answer = client.request("DELETE", "/api/v1/setup/sample-domain") or {}
        else:
            answer = client.request("GET", "/api/v1/setup/sample-domain") or {}
    except ApiError as error:
        if error.status != 409:
            raise
        print("hmctl: %s" % error.message, file=sys.stderr)
        return EXIT_REFUSED

    if arguments.json:
        render(answer, True)
        return EXIT_OK

    if arguments.verb in ("create", "remove"):
        # The message carries the password on a creation, because the server
        # mints one per sample and keeps only its hash: this line is the one
        # time anything says it.
        print(answer.get("message") or "")
        if arguments.verb == "create":
            print("The messages are delivered through the server's own path and arrive in a few seconds.")
            print("Remove the sample - hmctl sample-domain remove - before this server takes real mail.")
        return EXIT_OK

    state = answer.get("state", "")
    print("domain    %s" % answer.get("domain", ""))
    print("state     %s" % state)
    if state == "present":
        print("accounts  %s" % ", ".join(answer.get("accounts") or []))
        print("list      %s" % answer.get("list", ""))
        print("password  (shown once when the sample was made; not stored)")
        print("")
        print("Remove it - hmctl sample-domain remove - before this server takes real mail.")
    elif state == "foreign":
        print("")
        print("A domain of that name is on this server and the sample did not make it,")
        print("so create and remove both refuse. Delete it yourself if it is not wanted.")
    elif state == "unknown":
        print("")
        print("The server could not read its domain table, so it cannot say whether the")
        print("sample is here. create and remove both refuse until it can.")
    else:
        print("")
        print("No sample domain. hmctl sample-domain create makes one.")

    return EXIT_OK


def report_command(client, arguments):
    """The reports: a section of one, for a window and optionally one domain.

    The CSV is the SERVER'S csv, fetched with format=csv rather than written
    here out of the JSON. That is deliberate: two writers of the same table
    drift, and the one an administrator opens in a spreadsheet should be the
    one the route promises. It also means a column the route gains appears in
    the file without a change here.
    """
    if arguments.section == "sections":
        answer = client.request("GET", "/api/v1/reports")
        if arguments.json:
            render(answer, True)
            return EXIT_OK
        render(collection(answer, "sections"), False, ["name", "scope", "source", "summary"])
        print("")
        print("Sources:")
        for name, source in sorted((answer.get("sources") or {}).items()):
            print("  %-16s %-14s retention %s days, oldest %s"
                  % (name, "recording" if source.get("enabled") else "NOT RECORDING",
                     source.get("retention_days"), source.get("oldest") or "-"))
        for line in answer.get("not_answerable") or []:
            print("  cannot answer: %s" % line)
        return EXIT_OK

    query = []
    if arguments.set_from:
        query.append("from=" + urllib.parse.quote(arguments.set_from))
    if arguments.to:
        query.append("to=" + urllib.parse.quote(arguments.to))
    if arguments.domain:
        query.append("domain=" + urllib.parse.quote(arguments.domain))
    query.append("top=%d" % arguments.top)

    path = "/api/v1/reports/" + urllib.parse.quote(arguments.section)

    if arguments.csv:
        query.append("format=csv")
        # request() hands back a string when the body is not JSON, which a CSV
        # never is; anything else means the server answered a document rather
        # than a table and is written out as it came.
        text = client.request("GET", path + "?" + "&".join(query))
        if not isinstance(text, str):
            text = json.dumps(text, indent=2, sort_keys=True)
        if arguments.csv == "-":
            sys.stdout.write(text)
        else:
            with io.open(arguments.csv, "w", encoding="utf-8", newline="") as handle:
                handle.write(text)
            print("%s written to %s." % (arguments.section, arguments.csv))
        return EXIT_OK

    answer = client.request("GET", path + "?" + "&".join(query))
    if arguments.json:
        render(answer, True)
        return EXIT_OK

    if arguments.section == "summary":
        for name in sorted((answer.get("sections") or {}).keys()):
            part = answer["sections"][name]
            print("")
            print("== %s" % name)
            render(part.get("rows") or [], False, part.get("columns"))
        for skipped in answer.get("omitted") or []:
            print("")
            print("== %s: not shown - %s" % (skipped.get("section"), skipped.get("reason")))
        return EXIT_OK

    # The note is the honest half of a report: "the message trace is switched
    # off" is not the same statement as an empty table, and a client that
    # printed only the table would be hiding it.
    if answer.get("note"):
        print(answer["note"])
        print("")
    render(answer.get("rows") or [], False, answer.get("columns"))
    if answer.get("truncated"):
        print("")
        print("This answer was truncated: the window holds more rows than one report reads.")
    return EXIT_OK


def bounces_command(client, arguments):
    """How the server would read a bounce, and the suppression list's sweep.

    classify asks the server rather than holding a copy of the table here: the
    trust rule - a code the remote did not send itself is never a dead mailbox -
    is the part that matters, and a second implementation of it would drift from
    the one that decides.
    """
    if arguments.verb == "classify":
        body = {"status": arguments.status, "diagnostic": arguments.diagnostic or ""}
        answer = client.request("POST", "/api/v1/bounces/classify", body=body)
        if arguments.json:
            render(answer, True)
            return EXIT_OK
        print("%s: %s%s" % (answer.get("status") or arguments.status, answer.get("category"),
                            " - counts towards the suppression list" if answer.get("suppresses") else ""))
        if not answer.get("trusted"):
            print("(not trusted: the code was not the remote's own, so it is read by its words and never as a dead mailbox)")
        return EXIT_OK

    if arguments.verb == "expire":
        answer = client.request("POST", "/api/v1/bounces/expire")
        if arguments.json:
            render(answer, True)
            return EXIT_OK
        print("%d expired entr%s, %d old count%s and %d old report key%s removed."
              % (answer.get("suppressions_expired", 0), "y" if answer.get("suppressions_expired", 0) == 1 else "ies",
                 answer.get("counts_pruned", 0), "" if answer.get("counts_pruned", 0) == 1 else "s",
                 answer.get("report_keys_pruned", 0), "" if answer.get("report_keys_pruned", 0) == 1 else "s"))
        return EXIT_OK

    raise ApiError(0, "bounces takes classify or expire")


def openpgp_command(client, arguments):
    """The OpenPGP public keys this server publishes, and the Web Key Directory.

    A domain's listing shows the keys that are NOT being served beside the ones
    that are, which is the question this is usually run to answer. Only public
    keys are ever stored: a secret key is refused by the server with a sentence
    saying so, whether it arrives here, over COM or from the webmail.
    """
    if arguments.verb == "list":
        answer = client.request("GET", "/api/v1/domains/%s/openpgp" % arguments.domain) or {}
        if arguments.json:
            render(answer, True)
            return EXIT_OK
        if not answer.get("server_enabled"):
            print("The Web Key Directory is off at the server (settings openpgp web_key_directory_enabled).")
        elif not answer.get("domain_enabled"):
            print("%s does not publish keys (domain openpgp_publish)." % arguments.domain)
        render(collection(answer, "keys"), False,
               ["address", "fingerprint", "algorithm_name", "bits", "user_id", "published"])
        return EXIT_OK

    if arguments.verb == "show":
        render(client.request("GET", "/api/v1/accounts/%s/openpgp" % arguments.account), arguments.json)
        return EXIT_OK

    if arguments.verb == "store":
        # From a file, because a key is thousands of characters and a command
        # line is not where it belongs. "-" is standard input, so a key can be
        # piped straight out of gpg --armor --export.
        if arguments.file == "-":
            text = sys.stdin.read()
        else:
            with open(arguments.file, "r", encoding="utf-8", errors="replace") as handle:
                text = handle.read()
        if not text.strip():
            raise ApiError(0, "%s holds no key" % arguments.file)
        body = {"key": text}
        if arguments.allow_address_mismatch:
            body["allow_address_mismatch"] = True
        if arguments.publish:
            body["publish"] = True
        answer = client.request("PUT", "/api/v1/accounts/%s/openpgp" % arguments.account, body=body) or {}
        if arguments.json:
            render(answer, True)
        else:
            print("stored %s for %s%s" % (answer.get("fingerprint", ""), arguments.account,
                                          "; published" if arguments.publish else ""))
            if answer.get("advanced_url"):
                print(answer["advanced_url"])
        return EXIT_OK

    if arguments.verb == "remove":
        client.request("DELETE", "/api/v1/accounts/%s/openpgp/%s" % (arguments.account, arguments.fingerprint))
        print("removed %s from %s" % (arguments.fingerprint, arguments.account))
        return EXIT_OK

    if arguments.verb == "publish":
        body = {"openpgp_publish": arguments.state == "on"}
        answer = client.request("PUT", "/api/v1/accounts/%s" % arguments.account, body=body) or {}
        render(answer if arguments.json else {"address": arguments.account, "openpgp_publish": body["openpgp_publish"]},
               arguments.json)
        return EXIT_OK

    raise ApiError(0, "unknown openpgp verb")



def webhooks_command(client, arguments):
    """The two things an administrator does to a webhook that are not editing it:
    raise one event to see whether it arrives, and make the delivery pass happen
    now rather than within the minute.

    deliveries is here rather than as a resource verb because the queue is not a
    collection anybody edits - it is what the subscriptions have produced, and it
    is read with a filter or not at all.
    """
    if arguments.verb == "deliveries":
        query = []
        if arguments.webhook:
            query.append("webhook=" + urllib.parse.quote(arguments.webhook))
        if arguments.state:
            query.append("state=" + urllib.parse.quote(arguments.state))
        query.append("limit=%d" % arguments.limit)
        answer = client.request("GET", "/api/v1/webhooks/deliveries?" + "&".join(query))
        deliveries = answer.get("deliveries", []) if isinstance(answer, dict) else answer
        if arguments.json:
            render(answer, True)
            return EXIT_OK
        if not deliveries:
            print("Nothing has been queued.")
            return EXIT_OK
        render(deliveries, False, ["id", "webhook", "event", "state", "attempts", "error"])
        return EXIT_OK

    if arguments.verb == "run":
        answer = client.request("POST", "/api/v1/webhooks/run", body={})
        if arguments.json:
            render(answer, True)
            return EXIT_OK
        print("%d delivered, %d removed." % (answer.get("delivered", 0), answer.get("removed", 0)))
        return EXIT_OK

    if arguments.verb == "test":
        body = {}
        if arguments.event:
            body["event"] = arguments.event
        if arguments.domain:
            body["domain"] = arguments.domain
        if arguments.account:
            body["account"] = arguments.account
        answer = client.request("POST", "/api/v1/webhooks/test", body=body)
        if arguments.json:
            render(answer, True)
            return EXIT_OK
        queued = answer.get("queued", 0)
        print("Queued for %d subscription%s." % (queued, "" if queued == 1 else "s"))
        if not queued:
            print("(nothing subscribes to that event, the domain has webhooks off, "
                  "or every match is at its rate ceiling)")
        return EXIT_OK

    raise ApiError(0, "webhooks takes deliveries, run or test")


def complaints_command(client, arguments):
    """Every domain's complaint rate, and the feedback loops' sweep.

    The rate is asked of the server rather than worked out here from the
    listing: the window, the floor below which a rate is not judged, and the
    denominator are the server's, and a second implementation of them would
    drift from the one the alert fires on.
    """
    if arguments.verb == "rates":
        answer = client.request("GET", "/api/v1/complaints/rates")
        if arguments.json:
            render(answer, True)
            return EXIT_OK
        if not answer.get("processing_enabled"):
            print("Complaints are not being read; the rates below are whatever was recorded before.")
        rows = collection(answer, "domains")
        for row in rows:
            if row.get("judged"):
                print("%-40s %8d complaint(s) in %9d message(s)  %6d in a million"
                      % (row.get("domain"), row.get("complaints", 0), row.get("sent", 0), row.get("rate_ppm", 0)))
            else:
                print("%-40s %8d complaint(s) in %9d message(s)  (too little sent to judge)"
                      % (row.get("domain"), row.get("complaints", 0), row.get("sent", 0)))
        if not rows:
            print("No domain has sent or been complained about in the last %d days." % answer.get("window_days", 7))
        return EXIT_OK

    if arguments.verb == "expire":
        answer = client.request("POST", "/api/v1/complaints/expire")
        if arguments.json:
            render(answer, True)
            return EXIT_OK
        pruned = answer.get("complaints_pruned", 0)
        print("%d old complaint%s and tally row%s removed." % (pruned, "" if pruned == 1 else "s", "" if pruned == 1 else "s"))
        return EXIT_OK

    raise ApiError(0, "complaints takes rates or expire")


def dmarc_command(client, arguments):
    """The reports a domain receives, read: who sends as the domain and with
    what result, from the DMARC aggregate reports and the TLS-RPT reports the
    receivers of its mail posted to its report mailbox. The sums are the
    server's - a DMARC pass is an aligned DKIM or an aligned SPF, judged there
    once - and this only prints them.
    """
    if arguments.verb == "expire":
        if arguments.older_than_days is None:
            answer = client.request("POST", "/api/v1/domain-reports/expire")
        else:
            answer = client.request("POST", "/api/v1/domain-reports/expire", body={"older_than_days": arguments.older_than_days})
        if arguments.json:
            render(answer, True)
            return EXIT_OK
        if answer.get("kept_for_ever"):
            print("Nothing removed: domain_report_retention_days is 0, so the reports are kept for ever.")
        else:
            print("%d DMARC report(s) and %d TLS-RPT report(s) read more than %d days ago removed."
                  % (answer.get("dmarc_reports_pruned", 0), answer.get("tls_reports_pruned", 0), answer.get("older_than_days", 0)))
            if not answer.get("complete", True):
                print("More remain: the sweep stopped after its hundred passes of 5,000. Run it again to remove the rest.")
        return EXIT_OK

    base = "/api/v1/domains/%s" % urllib.parse.quote(arguments.domain, safe="")

    if arguments.verb == "summary":
        answer = client.request("GET", base + "/dmarc-reports/summary", query={"days": arguments.days})
        if arguments.json:
            render(answer, True)
            return EXIT_OK
        if not answer.get("mailbox_configured"):
            print("No report mailbox is named for %s, so no report is read: set report_mailbox on the domain and publish it as rua=."
                  % answer.get("domain", arguments.domain))
        print("%s over %d days: %d report(s) from %d reporter(s); %d message(s), %d passed DMARC, %d failed; "
              "%d quarantined, %d rejected by the receivers."
              % (answer.get("domain", arguments.domain), answer.get("days", arguments.days), answer.get("reports", 0),
                 len(collection(answer, "reporters") or []), answer.get("messages", 0), answer.get("passed", 0),
                 answer.get("failed", 0), answer.get("quarantined", 0), answer.get("rejected", 0)))
        sources = collection(answer, "sources")
        if sources:
            print_table(sources, ["ip", "header_from", "messages", "passed", "failed", "dkim_passed", "spf_passed",
                                  "quarantined", "rejected", "reporters"])
        else:
            print("No source has been reported in this window.")
        return EXIT_OK

    if arguments.verb == "reports":
        answer = client.request("GET", base + "/dmarc-reports", query={"limit": arguments.limit, "offset": arguments.offset})
        if arguments.json:
            render(answer, True)
            return EXIT_OK
        render(collection(answer, "items"), False,
               ["id", "org", "report_id", "begin", "end", "p", "sp", "pct", "messages", "passed", "records", "received",
                "message_from", "message_dmarc"])
        return EXIT_OK

    if arguments.verb == "report":
        answer = client.request("GET", base + "/dmarc-reports/%d" % arguments.id)
        if arguments.json:
            render(answer, True)
            return EXIT_OK
        print("Report %s from %s (%s), %s to %s, read %s: p=%s sp=%s adkim=%s aspf=%s pct=%s; %s message(s), %s passed DMARC."
              % (answer.get("report_id"), answer.get("org"), answer.get("email"), answer.get("begin"), answer.get("end"),
                 answer.get("received"), answer.get("p"), answer.get("sp"), answer.get("adkim"), answer.get("aspf"),
                 answer.get("pct"), answer.get("messages"), answer.get("passed")))
        render(collection(answer, "records"), False,
               ["source_ip", "count", "disposition", "dkim", "spf", "header_from", "envelope_from", "dkim_auth", "spf_auth"])
        return EXIT_OK

    if arguments.verb == "tls":
        answer = client.request("GET", base + "/tls-reports",
                                query={"days": arguments.days, "limit": arguments.limit, "offset": arguments.offset})
        if arguments.json:
            render(answer, True)
            return EXIT_OK
        print("%s over %d days: %d session(s) succeeded and %d failed, by what the senders that keep TLS-RPT reported."
              % (answer.get("domain", arguments.domain), answer.get("days", arguments.days),
                 answer.get("sessions_succeeded", 0), answer.get("sessions_failed", 0)))
        render(collection(answer, "items"), False,
               ["id", "org", "policy_type", "begin", "end", "successful", "failed", "received"])
        return EXIT_OK

    raise ApiError(0, "dmarc takes summary, reports, report, tls or expire")


def api_command(client, arguments):
    body = None
    if arguments.body:
        body = json.loads(sys.stdin.read()) if arguments.body == "-" else json.loads(arguments.body)
    answer = client.request(arguments.method.upper(), arguments.path, body=body)
    render(answer, True if arguments.json else False)
    return EXIT_OK


# ---------------------------------------------------------------------------
# The command line
# ---------------------------------------------------------------------------

def build_parser():
    parser = argparse.ArgumentParser(
        prog="hmctl",
        description="Administer an hMailServer over its REST API.",
        epilog="Environment: HMAIL_URL, HMAIL_API_KEY, HMAIL_PASSWORD, HMAIL_PASSWORD_FILE.",
    )
    parser.add_argument("--url", default=os.environ.get("HMAIL_URL", DEFAULT_URL),
                        help="the server's REST address (default %s)" % DEFAULT_URL)
    parser.add_argument("--api-key", default=os.environ.get("HMAIL_API_KEY"))
    parser.add_argument("--password", default=os.environ.get("HMAIL_PASSWORD"))
    parser.add_argument("--password-file", default=os.environ.get("HMAIL_PASSWORD_FILE"))
    parser.add_argument("--insecure", action="store_true",
                        help="accept a self-signed certificate; loopback only")
    parser.add_argument("--json", action="store_true", help="the server's JSON, unformatted by this tool")
    parser.add_argument("--timeout", type=int, default=30)
    parser.add_argument("--version", action="version", version="hmctl " + VERSION)

    # The same flags again, for every parser below, because `hmctl domain list
    # --json` is where a person types it and argparse takes the main parser's
    # options only before the subcommand. SUPPRESS is what makes that safe: an
    # inherited flag that was not given leaves no attribute, so it cannot
    # overwrite the value the main parser already read from the command line or
    # from the environment with its own default.
    common = argparse.ArgumentParser(add_help=False)
    common.add_argument("--url", default=argparse.SUPPRESS, help=argparse.SUPPRESS)
    common.add_argument("--api-key", default=argparse.SUPPRESS, help=argparse.SUPPRESS)
    common.add_argument("--password", default=argparse.SUPPRESS, help=argparse.SUPPRESS)
    common.add_argument("--password-file", default=argparse.SUPPRESS, help=argparse.SUPPRESS)
    common.add_argument("--insecure", action="store_true", default=argparse.SUPPRESS, help=argparse.SUPPRESS)
    common.add_argument("--json", action="store_true", default=argparse.SUPPRESS,
                        help="the server's JSON, unformatted by this tool")
    common.add_argument("--timeout", type=int, default=argparse.SUPPRESS, help=argparse.SUPPRESS)

    commands = parser.add_subparsers(dest="command")

    status = commands.add_parser("status", parents=[common], help="what the server says about itself")
    status.set_defaults(handler=status_command)

    resource_verbs = {}
    for name in sorted(RESOURCES):
        resource = RESOURCES[name]
        sub = commands.add_parser(name, help=resource["help"])
        sub.set_defaults(handler=resource_command, resource=name)
        verbs = sub.add_subparsers(dest="verb")
        resource_verbs[name] = verbs
        for verb in resource["verbs"]:
            verb_parser = verbs.add_parser(verb, parents=[common])
            if verb in ("get", "update", "delete"):
                verb_parser.add_argument("id", help="the %s" % resource["id"])
            if verb in ("create", "update"):
                verb_parser.add_argument("--set", action="append", metavar="NAME=VALUE",
                                         help="a field; true, false, null, numbers and JSON are read as such")
                verb_parser.add_argument("--set-string", action="append", metavar="NAME=VALUE",
                                         help="a field whose value stays a string")
                verb_parser.add_argument("--from-json", metavar="FILE", help="a JSON object, or - for stdin")
            if resource.get("parent"):
                verb_parser.add_argument("--" + resource["parent"],
                                         help="the %s these belong to" % resource["parent"])
            if verb == "list" and resource.get("search"):
                verb_parser.add_argument("--search", help="only the addresses containing this")

    # One value, three levels: a domain's or an account's own value of a setting
    # that may be set below the server, or inherit to take the level above.
    for name, what in (("domain", "the domain's name"), ("account", "the account's address")):
        level = resource_verbs[name].add_parser(
            "set", parents=[common],
            help="its own value of a setting that may be set below the server, or inherit (GET /api/v1/settings/levels lists them)")
        level.add_argument("id", help=what)
        level.add_argument("setting", help="the setting's name, as REST spells it: recall_days, max_message_size_kb, ...")
        level.add_argument("value", help="the value (true, false and numbers are read as such), or inherit")
        level.set_defaults(handler=level_set_command)

    queue = commands.add_parser("queue", help="the delivery queue")
    queue.set_defaults(handler=queue_command)
    queue_verbs = queue.add_subparsers(dest="verb")
    queue_verbs.add_parser("list", parents=[common])
    for verb in ("retry", "delete"):
        p = queue_verbs.add_parser(verb, parents=[common])
        p.add_argument("id")

    quarantine = commands.add_parser("quarantine", help="held mail")
    quarantine.set_defaults(handler=quarantine_command)
    quarantine_verbs = quarantine.add_subparsers(dest="verb")
    quarantine_verbs.add_parser("list", parents=[common])
    for verb in ("release", "delete"):
        p = quarantine_verbs.add_parser(verb, parents=[common])
        p.add_argument("id")

    reputation = commands.add_parser("reputation",
                                     help="this server's own addresses against the DNS block lists")
    reputation.set_defaults(handler=reputation_command)
    reputation_verbs = reputation.add_subparsers(dest="verb")
    reputation_verbs.add_parser("status", parents=[common],
                                help="what the last check found; exits 1 when an address is listed, when a list could not be asked, and - while the watch is on - when a pair has not been checked yet")
    reputation_verbs.add_parser("check", parents=[common],
                                help="check now rather than waiting for the daily check; server administrators")

    privacy = commands.add_parser("privacy", help="a person's data: everything the server holds as one archive, and every trace erased")
    privacy.set_defaults(handler=privacy_command)
    privacy_verbs = privacy.add_subparsers(dest="verb")
    export_parser = privacy_verbs.add_parser("export", parents=[common],
                                             help="write everything the server holds on the account to one .zip on the server's disk")
    export_parser.add_argument("address")
    export_parser.add_argument("--archive", required=True, metavar="FILE", help="the path of the .zip to write, on the server")
    erase_parser = privacy_verbs.add_parser("erase", parents=[common],
                                            help="erase every stored trace of an address that deleting its account leaves")
    erase_parser.add_argument("address")
    erase_parser.add_argument("--archive", action="store_true", help="the address's archive tree too, copies under hold excepted")
    erase_parser.add_argument("--logs", action="store_true", help="rewrite the log files so no line names the address")

    settings = commands.add_parser("settings", help="the server settings, by group")
    settings.set_defaults(handler=settings_command)
    settings_verbs = settings.add_subparsers(dest="verb")
    get = settings_verbs.add_parser("get", parents=[common])
    get.add_argument("group", nargs="?", choices=SETTING_GROUPS, default="settings")
    get.add_argument("name", nargs="?", help="one setting rather than the group")
    set_parser = settings_verbs.add_parser("set", parents=[common])
    set_parser.add_argument("group", nargs="?", choices=SETTING_GROUPS, default="settings")
    set_parser.add_argument("--set", action="append", metavar="NAME=VALUE", required=False)
    set_parser.add_argument("--set-string", action="append", metavar="NAME=VALUE")
    set_parser.add_argument("--from-json", metavar="FILE")
    set_parser.set_defaults(name=None)
    breach_test = settings_verbs.add_parser("breach-test", parents=[common],
                                            help="ask the password breach check's endpoint for a range, changing nothing")
    breach_test.add_argument("--endpoint", metavar="URL", help="the endpoint to test; the configured breach_check_url when left out")
    breach_test.set_defaults(group=None, name=None)
    # The setting profiles (docs/Profiles.md).
    profiles_parser = settings_verbs.add_parser("profiles", parents=[common],
                                                help="the four setting profiles, how much of each would change here, and the one applied")
    profiles_parser.set_defaults(group=None, name=None)
    profile_parser = settings_verbs.add_parser("profile", parents=[common],
                                               help="one profile's diff: every setting beside the value this server holds")
    profile_parser.add_argument("name", help="office, gateway, relay or sender")
    profile_parser.set_defaults(group=None)
    apply_profile_parser = settings_verbs.add_parser("apply-profile", parents=[common],
                                                     help="apply a profile; what it replaces is kept so that it can be put back")
    apply_profile_parser.add_argument("name", help="office, gateway, relay or sender")
    apply_profile_parser.add_argument("--confirm", metavar="TOKEN",
                                      help="apply an entry that closes a listener, trusts an address range, turns a filter or a check off, or shortens how long data is kept, confirming the diff whose token hmctl settings profile NAME printed; refused without it")
    apply_profile_parser.add_argument("--token", metavar="TOKEN",
                                      help="the diff's token, for an apply that needs no confirmation: refused when the settings moved since the diff was read")
    apply_profile_parser.set_defaults(group=None)
    revert_profile_parser = settings_verbs.add_parser("revert-profile", parents=[common],
                                                      help="put back what the last profile changed")
    revert_profile_parser.set_defaults(group=None, name=None)
    discard_profile_parser = settings_verbs.add_parser("discard-profile", parents=[common],
                                                       help="keep the last profile's settings and forget the way back")
    discard_profile_parser.set_defaults(group=None, name=None)
    decline_profile_parser = settings_verbs.add_parser("decline-profile", parents=[common],
                                                       help="keep the settings this version ships with: record that no profile is wanted")
    decline_profile_parser.set_defaults(group=None, name=None)

    ini = commands.add_parser("ini", help="the server settings that were hMailServer.ini's [Settings] keys, stored in the database")
    ini.set_defaults(handler=ini_command)
    ini_verbs = ini.add_subparsers(dest="verb")
    ini_verbs.add_parser("list", parents=[common])
    for verb in ("get", "delete"):
        p = ini_verbs.add_parser(verb, parents=[common])
        p.add_argument("name")
    p = ini_verbs.add_parser("set", parents=[common])
    p.add_argument("name")
    p.add_argument("value")

    log = commands.add_parser("log", help="the log files")
    log.set_defaults(handler=log_command)
    log_verbs = log.add_subparsers(dest="verb")
    log_verbs.add_parser("list", parents=[common])
    tail = log_verbs.add_parser("tail", parents=[common])
    tail.add_argument("name")
    tail.add_argument("--lines", type=int, default=50)

    backup = commands.add_parser("backup", help="the server's own backup, and putting one back")
    backup.set_defaults(handler=backup_command)
    backup_verbs = backup.add_subparsers(dest="verb")
    backup_verbs.add_parser("status", parents=[common])
    backup_verbs.add_parser("start", parents=[common])

    backup_restore = backup_verbs.add_parser(
        "restore", parents=[common],
        help="put an archive back - destructive: it replaces what it restores")
    backup_restore.add_argument("file", help="the archive's path ON THE SERVER")
    backup_restore.add_argument("--settings", action="store_true", help="restore the server settings")
    backup_restore.add_argument("--domains", action="store_true", help="restore the domains, accounts and aliases")
    backup_restore.add_argument("--messages", action="store_true", help="restore the message files (with --domains)")
    backup_restore.add_argument("--passphrase-file", metavar="FILE",
                                help="a file holding the archive's passphrase, or - for standard input")
    backup_restore.add_argument("--ask", dest="ask_passphrase", action="store_true",
                                help="ask for the archive's passphrase here, not echoed")

    backup_passphrase = backup_verbs.add_parser(
        "passphrase", parents=[common],
        help="the passphrase the backup archive is encrypted with")
    backup_passphrase.add_argument("action", choices=["set", "clear"])
    backup_passphrase.add_argument("--passphrase-file", metavar="FILE",
                                   help="a file holding the passphrase, or - for standard input")
    backup_passphrase.add_argument("--ask", dest="ask_passphrase", action="store_true",
                                   help="ask for the passphrase here, twice, not echoed")

    # One mailbox, or one message, from an archive in the backup destination.
    backup_verbs.add_parser("archives", parents=[common], help="the archives in the backup destination, newest first")

    backup_mailbox = backup_verbs.add_parser(
        "mailbox", parents=[common],
        help="one account's folders and messages as an archive holds them, with the ids restore-mailbox takes")
    backup_mailbox.add_argument("archive", help="the archive's name as `backup archives` lists it (a path is not taken here; over COM it is)")
    backup_mailbox.add_argument("address", help="the account")
    backup_mailbox.add_argument("--folder", help="list one folder path only")
    backup_mailbox.add_argument("--passphrase-file", metavar="FILE",
                                help="a file holding an encrypted archive's passphrase, or - for standard input")
    backup_mailbox.add_argument("--ask", dest="ask_passphrase", action="store_true",
                                help="ask for the archive's passphrase here, not echoed")

    backup_restore_mailbox = backup_verbs.add_parser(
        "restore-mailbox", parents=[common],
        help="put one account's mailbox back from an archive (merge, the default, or replace), or one message of it")
    backup_restore_mailbox.add_argument("archive", help="the archive's name as `backup archives` lists it (a path is not taken here; over COM it is)")
    backup_restore_mailbox.add_argument("address", help="the account; it has to exist on the server")
    backup_restore_mailbox.add_argument("--merge", action="store_true",
                                        help="add the archive's folders and messages the mailbox lacks; nothing there is removed (the default)")
    backup_restore_mailbox.add_argument("--replace", action="store_true",
                                        help="empty every live folder first, so the mailbox is the archive's - mail since the backup is gone")
    backup_restore_mailbox.add_argument("--message", type=int, metavar="ID",
                                        help="restore only this message, by the id `backup mailbox` lists")
    backup_restore_mailbox.add_argument("--folder", help="with --message: the live folder to put it in (default: the one it was in)")
    backup_restore_mailbox.add_argument("--passphrase-file", metavar="FILE",
                                        help="a file holding an encrypted archive's passphrase, or - for standard input")
    backup_restore_mailbox.add_argument("--ask", dest="ask_passphrase", action="store_true",
                                        help="ask for the archive's passphrase here, not echoed")

    backup_restore_status = backup_verbs.add_parser(
        "restore-status", parents=[common], help="what became of a mailbox restore, by its job number")
    backup_restore_status.add_argument("job", type=int)

    match = commands.add_parser("match", parents=[common], help="try a rule criterion against a value")
    match.set_defaults(handler=rule_match_command)
    match.add_argument("type", choices=["equals", "contains", "less_than", "greater_than",
                                        "regex", "not_contains", "not_equals", "wildcard"])
    match.add_argument("value", help="the criterion's value, as a rule stores it")
    match.add_argument("against", help="the text to try it against")

    app = commands.add_parser("app-password", help="an account's application passwords")
    app.set_defaults(handler=apppassword_command)
    app.add_argument("address")
    app_verbs = app.add_subparsers(dest="verb")
    app_verbs.add_parser("list", parents=[common])
    create = app_verbs.add_parser("create", parents=[common])
    create.add_argument("name")
    delete = app_verbs.add_parser("delete", parents=[common])
    delete.add_argument("id")

    device = commands.add_parser("device", help="the clients that have signed in to an account, and revoking one")
    device.set_defaults(handler=device_command)
    device.add_argument("address")
    device_verbs = device.add_subparsers(dest="verb")
    device_verbs.add_parser("list", parents=[common])
    revoke = device_verbs.add_parser("revoke", parents=[common])
    revoke.add_argument("id", help="the client's id, thirty-two hex characters from the list")
    revoke.add_argument("--credential", action="store_true",
                        help="also turn off the app password the client signed in with, where it used one")
    revoke.add_argument("--forget", action="store_true", help="remove the entry rather than marking it revoked")

    member = commands.add_parser("member", help="the accounts in a group")
    member.set_defaults(handler=member_command)
    member.add_argument("group", help="the group's id")
    member_verbs = member.add_subparsers(dest="verb")
    member_verbs.add_parser("list", parents=[common])
    for verb in ("add", "remove"):
        p = member_verbs.add_parser(verb, parents=[common])
        p.add_argument("account", help="an address, or an account id")

    # The schedule itself is five keys of the domain:
    #   hmctl domain update example.com --set dkim_rotation_enabled=true
    dkim = commands.add_parser("dkim", help="a domain's DKIM key rotation: the records to publish, and rotate, switch, retire")
    dkim.set_defaults(handler=dkim_command)
    dkim_verbs = dkim.add_subparsers(dest="verb")
    for verb, text in (("status", "the rotation's state and the records to publish, keep or remove"),
                       ("rotate", "generate and stage the next key now"),
                       ("check", "look the records up from the server's resolver now"),
                       ("switch", "move signing to the next key"),
                       ("retire", "retire the old selector now, before its retention period ends"),
                       ("cancel", "cancel the next key; signing does not change")):
        p = dkim_verbs.add_parser(verb, parents=[common], help=text)
        p.add_argument("domain")
        if verb == "rotate":
            p.add_argument("--selector", help="the new key's selector (default: s and today's UTC date)")
        if verb == "switch":
            p.add_argument("--force", action="store_true",
                           help="do not wait out the propagation period; the record must still resolve")
    run = dkim_verbs.add_parser("run", parents=[common], help="one pass of the rotation over every domain, now")
    run.add_argument("--seconds-ahead", type=int, default=0,
                     help="move the clock forward for this pass (tests only: a switch or retire it makes is real "
                          "and early, and the times it records stay recorded; server administrators)")

    openpgp = commands.add_parser("openpgp", help="the OpenPGP public keys this server publishes (Web Key Directory)")
    openpgp.set_defaults(handler=openpgp_command)
    openpgp_verbs = openpgp.add_subparsers(dest="verb")
    pgp_list = openpgp_verbs.add_parser("list", parents=[common], help="every key held for a domain's accounts")
    pgp_list.add_argument("domain")
    pgp_show = openpgp_verbs.add_parser("show", parents=[common], help="one account's keys and where a client looks for them")
    pgp_show.add_argument("account")
    pgp_store = openpgp_verbs.add_parser("store", parents=[common], help="store a PUBLIC key for an account from a file")
    pgp_store.add_argument("account")
    pgp_store.add_argument("file", help="the armoured key, or - for standard input (gpg --armor --export)")
    pgp_store.add_argument("--publish", action="store_true", help="switch this account's publishing on as well")
    pgp_store.add_argument("--allow-address-mismatch", action="store_true",
                           help="store a key whose user ids do not carry the account's own address (server administrators)")
    pgp_remove = openpgp_verbs.add_parser("remove", parents=[common], help="remove one key from an account")
    pgp_remove.add_argument("account")
    pgp_remove.add_argument("fingerprint")
    pgp_publish = openpgp_verbs.add_parser("publish", parents=[common], help="publish an account's keys, or stop")
    pgp_publish.add_argument("account")
    pgp_publish.add_argument("state", choices=["on", "off"])

    mailbox = commands.add_parser("mailbox", help="rename a mailbox, or move it to another domain on this server")
    mailbox.set_defaults(handler=mailbox_command)
    mailbox_verbs = mailbox.add_subparsers(dest="verb")
    status_parser = mailbox_verbs.add_parser("status", parents=[common],
                                             help="whether the mailbox can be moved now, and what it has been through")
    status_parser.add_argument("address")
    move = mailbox_verbs.add_parser("move", parents=[common],
                                    help="rename it, or move it to another domain; the mailbox, its messages and everything keyed on the account go with it")
    move.add_argument("address")
    move.add_argument("new_address", metavar="new-address",
                      help="the address it takes; the domain in it may be any domain this server holds")
    move.add_argument("--drop-alias", action="store_true",
                      help="do not keep the old address as an alias; mail to it is refused from then on")

    accounts = commands.add_parser("accounts", help="accounts in bulk, from and to a CSV file")
    accounts_verbs = accounts.add_subparsers(dest="verb")
    importer = accounts_verbs.add_parser("import", parents=[common])
    importer.set_defaults(handler=accounts_import)
    importer.add_argument("file")
    importer.add_argument("--domain", action="append",
                          help="the domain(s) to read existing addresses from; derived from the file when omitted")
    importer.add_argument("--dry-run", action="store_true", help="say what would happen and change nothing")
    exporter = accounts_verbs.add_parser("export", parents=[common])
    exporter.set_defaults(handler=accounts_export)
    exporter.add_argument("domain")
    exporter.add_argument("file", nargs="?", default="-")

    traphits = commands.add_parser("traphits", help="what has addressed a spam trap")
    traphits.set_defaults(handler=traphits_command)
    traphits_verbs = traphits.add_subparsers(dest="verb")
    hits_list = traphits_verbs.add_parser("list", parents=[common])
    hits_list.add_argument("--domain", help="one domain's hits; every domain's when left out")
    hits_list.add_argument("--limit", type=int, default=None, metavar="N",
                           help="how many rows, 1 to 1000 (100 by default)")
    hits_clear = traphits_verbs.add_parser("clear", parents=[common],
                                           help="clear the record; the bans and blocked senders it made stay")
    hits_clear.set_defaults(domain=None, limit=None)

    sample = commands.add_parser("sample-domain",
                                 help="a throwaway example.test with three mailboxes, a list, a rule and some delivered mail")
    sample.set_defaults(handler=sample_domain_command)
    sample_verbs = sample.add_subparsers(dest="verb")
    sample_verbs.add_parser("status", parents=[common],
                            help="whether the sample domain is there, and what is in it")
    sample_verbs.add_parser("create", parents=[common],
                            help="make it; refused when a domain named example.test is already here")
    sample_verbs.add_parser("remove", parents=[common],
                            help="remove it and say what went; never removes an example.test this did not make")

    configuration = commands.add_parser("config", help="the whole configuration as one file")
    configuration_verbs = configuration.add_subparsers(dest="verb")
    exporter = configuration_verbs.add_parser("export", parents=[common])
    exporter.set_defaults(handler=config_export)
    exporter.add_argument("file", nargs="?", default="-")
    changed = configuration_verbs.add_parser("changed", parents=[common],
                                             help="every setting whose value is not the shipped default, with the default beside it")
    changed.set_defaults(handler=config_changed)
    differ = configuration_verbs.add_parser("diff", parents=[common])
    differ.set_defaults(handler=config_diff)
    differ.add_argument("file")
    applier = configuration_verbs.add_parser("apply", parents=[common])
    applier.set_defaults(handler=config_apply)
    applier.add_argument("file")
    applier.add_argument("--force", action="store_true", help="apply the plan; without it nothing is changed")
    applier.add_argument("--allow-delete", action="store_true",
                         help="remove what the server has and the document does not; off by default")
    report = commands.add_parser("report", parents=[common],
                                 help="what happened on this server, per domain and per day")
    report.set_defaults(handler=report_command)
    report.add_argument("section", choices=["sections", "summary", "traffic", "failures", "senders",
                                            "recipients", "mailboxes", "volume", "storage", "countries"],
                        help="sections lists what can be reported and what cannot")
    report.add_argument("--from", dest="set_from", metavar="YYYY-MM-DD",
                        help="the first day of the window; the default is 30 days back")
    report.add_argument("--to", metavar="YYYY-MM-DD", help="the last day; the default is today on the server")
    report.add_argument("--domain", help="one local domain; the default is every domain the credential may see")
    report.add_argument("--top", type=int, default=10, metavar="N",
                        help="how many rows the senders, recipients and mailboxes sections return (1-200)")
    report.add_argument("--csv", metavar="FILE", nargs="?", const="-",
                        help="ask the server for the table as CSV and write it here, or to stdout")

    bounces = commands.add_parser("bounces", help="how a bounce is read, and the suppression list's sweep")
    bounces.set_defaults(handler=bounces_command)
    bounces_verbs = bounces.add_subparsers(dest="verb")
    classify = bounces_verbs.add_parser("classify", parents=[common])
    classify.add_argument("status", help="an enhanced status code (5.1.1) or a reply (\"550 5.1.1 User unknown\")")
    classify.add_argument("diagnostic", nargs="?", help="a delivery report's Diagnostic-Code, when status is its Status field")
    bounces_verbs.add_parser("expire", parents=[common])

    webhooks = commands.add_parser("webhooks", help="the webhook delivery queue, a pass, and a test event")
    webhooks.set_defaults(handler=webhooks_command)
    webhooks_verbs = webhooks.add_subparsers(dest="verb")
    deliveries = webhooks_verbs.add_parser("deliveries", parents=[common])
    deliveries.add_argument("--webhook", help="only this subscription's")
    deliveries.add_argument("--state", choices=["pending", "delivered", "dead-lettered"])
    deliveries.add_argument("--limit", type=int, default=100, help="rows to return, 1 to 1000")
    webhooks_verbs.add_parser("run", parents=[common])
    test = webhooks_verbs.add_parser("test", parents=[common])
    test.add_argument("--event", choices=["message.received", "message.delivered", "message.bounced",
                                          "message.quarantined", "account.locked"],
                      help="the event to raise; message.received by default")
    test.add_argument("--domain", help="the domain the event is about, so a confined subscription matches it")
    test.add_argument("--account", help="the address the event is about")

    complaints = commands.add_parser("complaints", help="the complaint rate of each domain, and the feedback loops' sweep")
    complaints.set_defaults(handler=complaints_command)
    complaints_verbs = complaints.add_subparsers(dest="verb")
    complaints_verbs.add_parser("rates", parents=[common])
    complaints_verbs.add_parser("expire", parents=[common])

    dmarc = commands.add_parser("dmarc", help="the DMARC aggregate and TLS-RPT reports a domain receives: who sends as it, and with what result")
    dmarc.set_defaults(handler=dmarc_command)
    dmarc_verbs = dmarc.add_subparsers(dest="verb")
    for verb, text in (("summary", "who sends as the domain over a window, with the totals, per source"),
                       ("reports", "the aggregate reports read for the domain, newest first"),
                       ("report", "one report with its records, by the id the listing shows"),
                       ("tls", "the TLS-RPT reports read for the domain over a window")):
        p = dmarc_verbs.add_parser(verb, parents=[common], help=text)
        p.add_argument("domain")
        if verb == "report":
            p.add_argument("id", type=int)
        if verb in ("summary", "tls"):
            p.add_argument("--days", type=int, default=30, help="the window, in days back from now (default 30)")
        if verb in ("reports", "tls"):
            p.add_argument("--limit", type=int, default=50, help="rows to return, 1 to 1000")
            p.add_argument("--offset", type=int, default=0, help="rows to skip")
    expire = dmarc_verbs.add_parser("expire", parents=[common], help="run the reports' retention sweep now")
    expire.add_argument("--older-than-days", type=int, dest="older_than_days",
                        help="use this instead of domain_report_retention_days; 0 removes every report of every domain")
    api = commands.add_parser("api", parents=[common], help="any method on any path - the escape hatch")
    api.set_defaults(handler=api_command)
    api.add_argument("method")
    api.add_argument("path")
    api.add_argument("body", nargs="?", help="a JSON body, or - for stdin")

    return parser


def can_prompt():
    """Whether there is a person at a console to answer a password prompt.

    Not sys.stdin.isatty() alone, which is wrong on Windows in the one case
    that matters: the NUL device is a character device, so isatty() answers
    True for it, and a scheduled task or a pipeline that redirected stdin from
    nul would sit at a prompt nobody can see until something killed it. That is
    the worst failure a small tool can have - silence - and it is exactly what
    a check of this file caught.

    GetConsoleMode tells the two apart: it succeeds only for a real console
    handle. Anywhere else isatty() is the right question.
    """
    if os.name != "nt":
        return sys.stdin.isatty()

    try:
        import ctypes
        import msvcrt

        handle = msvcrt.get_osfhandle(sys.stdin.fileno())
        mode = ctypes.c_uint()
        return bool(ctypes.windll.kernel32.GetConsoleMode(ctypes.c_void_p(handle), ctypes.byref(mode)))
    except Exception:
        return False


def resolve_password(arguments):
    if arguments.api_key:
        return None
    if arguments.password:
        return arguments.password
    if arguments.password_file:
        with io.open(arguments.password_file, encoding="utf-8") as handle:
            return handle.read().strip()
    if not can_prompt():
        raise ApiError(0, "no credential: give --api-key, --password, --password-file, or one of "
                          "HMAIL_API_KEY, HMAIL_PASSWORD, HMAIL_PASSWORD_FILE")
    return getpass.getpass("Administrator password for %s: " % arguments.url)


def main(argv=None):
    parser = build_parser()
    arguments = parser.parse_args(argv)

    if not getattr(arguments, "command", None):
        parser.print_help()
        return EXIT_USAGE
    if not getattr(arguments, "handler", None) or (
            "verb" in vars(arguments) and arguments.verb is None):
        parser.parse_args([arguments.command, "--help"])
        return EXIT_USAGE

    # SUPPRESS leaves an inherited flag out of the namespace when it was not
    # given after the subcommand, so every one of them is read with the main
    # parser's value behind it.
    for name, fallback in (("url", DEFAULT_URL), ("api_key", None), ("password", None),
                           ("password_file", None), ("insecure", False), ("json", False), ("timeout", 30)):
        if not hasattr(arguments, name):
            setattr(arguments, name, fallback)

    try:
        password = resolve_password(arguments)
        client = Client(arguments.url, api_key=arguments.api_key, password=password,
                        insecure=arguments.insecure, timeout=arguments.timeout)
        return arguments.handler(client, arguments)
    except ApiError as error:
        if error.status == 401:
            print("hmctl: the server refused the credential (401). Check the administrator password or the API key.",
                  file=sys.stderr)
            return EXIT_AUTH
        if error.status == 403:
            print("hmctl: forbidden (403): %s" % error.message, file=sys.stderr)
            return EXIT_AUTH
        if error.status == 404:
            print("hmctl: not found (404): %s" % error.message, file=sys.stderr)
            return EXIT_NOTFOUND
        print("hmctl: %s" % error.message, file=sys.stderr)
        return EXIT_ERROR
    except KeyboardInterrupt:
        return EXIT_ERROR
    except (OSError, IOError, ValueError) as error:
        print("hmctl: %s" % error, file=sys.stderr)
        return EXIT_ERROR


if __name__ == "__main__":
    sys.exit(main())
