Init: mediaserver

This commit is contained in:
2023-02-08 12:13:28 +01:00
parent 848bc9739c
commit f7c23d4ba9
31914 changed files with 6175775 additions and 0 deletions

View File

@@ -0,0 +1,12 @@
# flake8: noqa
from . import environment
from .config import ConfigurationError
from .config import DOCKER_CONFIG_KEYS
from .config import find
from .config import is_url
from .config import load
from .config import merge_environment
from .config import merge_labels
from .config import parse_environment
from .config import parse_labels
from .config import resolve_build_args

View File

@@ -0,0 +1,812 @@
{
"$schema": "http://json-schema.org/draft/2019-09/schema#",
"id": "compose_spec.json",
"type": "object",
"title": "Compose Specification",
"description": "The Compose file is a YAML file defining a multi-containers based application.",
"properties": {
"version": {
"type": "string",
"description": "Version of the Compose specification used. Tools not implementing required version MUST reject the configuration file."
},
"services": {
"id": "#/properties/services",
"type": "object",
"patternProperties": {
"^[a-zA-Z0-9._-]+$": {
"$ref": "#/definitions/service"
}
},
"additionalProperties": false
},
"networks": {
"id": "#/properties/networks",
"type": "object",
"patternProperties": {
"^[a-zA-Z0-9._-]+$": {
"$ref": "#/definitions/network"
}
}
},
"volumes": {
"id": "#/properties/volumes",
"type": "object",
"patternProperties": {
"^[a-zA-Z0-9._-]+$": {
"$ref": "#/definitions/volume"
}
},
"additionalProperties": false
},
"secrets": {
"id": "#/properties/secrets",
"type": "object",
"patternProperties": {
"^[a-zA-Z0-9._-]+$": {
"$ref": "#/definitions/secret"
}
},
"additionalProperties": false
},
"configs": {
"id": "#/properties/configs",
"type": "object",
"patternProperties": {
"^[a-zA-Z0-9._-]+$": {
"$ref": "#/definitions/config"
}
},
"additionalProperties": false
}
},
"patternProperties": {"^x-": {}},
"additionalProperties": false,
"definitions": {
"service": {
"id": "#/definitions/service",
"type": "object",
"properties": {
"deploy": {"$ref": "#/definitions/deployment"},
"build": {
"oneOf": [
{"type": "string"},
{
"type": "object",
"properties": {
"context": {"type": "string"},
"dockerfile": {"type": "string"},
"args": {"$ref": "#/definitions/list_or_dict"},
"labels": {"$ref": "#/definitions/list_or_dict"},
"cache_from": {"type": "array", "items": {"type": "string"}},
"network": {"type": "string"},
"target": {"type": "string"},
"shm_size": {"type": ["integer", "string"]},
"extra_hosts": {"$ref": "#/definitions/list_or_dict"},
"isolation": {"type": "string"}
},
"additionalProperties": false,
"patternProperties": {"^x-": {}}
}
]
},
"blkio_config": {
"type": "object",
"properties": {
"device_read_bps": {
"type": "array",
"items": {"$ref": "#/definitions/blkio_limit"}
},
"device_read_iops": {
"type": "array",
"items": {"$ref": "#/definitions/blkio_limit"}
},
"device_write_bps": {
"type": "array",
"items": {"$ref": "#/definitions/blkio_limit"}
},
"device_write_iops": {
"type": "array",
"items": {"$ref": "#/definitions/blkio_limit"}
},
"weight": {"type": "integer"},
"weight_device": {
"type": "array",
"items": {"$ref": "#/definitions/blkio_weight"}
}
},
"additionalProperties": false
},
"cap_add": {"type": "array", "items": {"type": "string"}, "uniqueItems": true},
"cap_drop": {"type": "array", "items": {"type": "string"}, "uniqueItems": true},
"cgroup_parent": {"type": "string"},
"command": {
"oneOf": [
{"type": "string"},
{"type": "array", "items": {"type": "string"}}
]
},
"configs": {
"type": "array",
"items": {
"oneOf": [
{"type": "string"},
{
"type": "object",
"properties": {
"source": {"type": "string"},
"target": {"type": "string"},
"uid": {"type": "string"},
"gid": {"type": "string"},
"mode": {"type": "number"}
},
"additionalProperties": false,
"patternProperties": {"^x-": {}}
}
]
}
},
"container_name": {"type": "string"},
"cpu_count": {"type": "integer", "minimum": 0},
"cpu_percent": {"type": "integer", "minimum": 0, "maximum": 100},
"cpu_shares": {"type": ["number", "string"]},
"cpu_quota": {"type": ["number", "string"]},
"cpu_period": {"type": ["number", "string"]},
"cpu_rt_period": {"type": ["number", "string"]},
"cpu_rt_runtime": {"type": ["number", "string"]},
"cpus": {"type": ["number", "string"]},
"cpuset": {"type": "string"},
"credential_spec": {
"type": "object",
"properties": {
"config": {"type": "string"},
"file": {"type": "string"},
"registry": {"type": "string"}
},
"additionalProperties": false,
"patternProperties": {"^x-": {}}
},
"depends_on": {
"oneOf": [
{"$ref": "#/definitions/list_of_strings"},
{
"type": "object",
"additionalProperties": false,
"patternProperties": {
"^[a-zA-Z0-9._-]+$": {
"type": "object",
"additionalProperties": false,
"properties": {
"condition": {
"type": "string",
"enum": ["service_started", "service_healthy", "service_completed_successfully"]
}
},
"required": ["condition"]
}
}
}
]
},
"device_cgroup_rules": {"$ref": "#/definitions/list_of_strings"},
"devices": {"type": "array", "items": {"type": "string"}, "uniqueItems": true},
"dns": {"$ref": "#/definitions/string_or_list"},
"dns_opt": {"type": "array","items": {"type": "string"}, "uniqueItems": true},
"dns_search": {"$ref": "#/definitions/string_or_list"},
"domainname": {"type": "string"},
"entrypoint": {
"oneOf": [
{"type": "string"},
{"type": "array", "items": {"type": "string"}}
]
},
"env_file": {"$ref": "#/definitions/string_or_list"},
"environment": {"$ref": "#/definitions/list_or_dict"},
"expose": {
"type": "array",
"items": {
"type": ["string", "number"],
"format": "expose"
},
"uniqueItems": true
},
"extends": {
"oneOf": [
{"type": "string"},
{
"type": "object",
"properties": {
"service": {"type": "string"},
"file": {"type": "string"}
},
"required": ["service"],
"additionalProperties": false
}
]
},
"external_links": {"type": "array", "items": {"type": "string"}, "uniqueItems": true},
"extra_hosts": {"$ref": "#/definitions/list_or_dict"},
"group_add": {
"type": "array",
"items": {
"type": ["string", "number"]
},
"uniqueItems": true
},
"healthcheck": {"$ref": "#/definitions/healthcheck"},
"hostname": {"type": "string"},
"image": {"type": "string"},
"init": {"type": "boolean"},
"ipc": {"type": "string"},
"isolation": {"type": "string"},
"labels": {"$ref": "#/definitions/list_or_dict"},
"links": {"type": "array", "items": {"type": "string"}, "uniqueItems": true},
"logging": {
"type": "object",
"properties": {
"driver": {"type": "string"},
"options": {
"type": "object",
"patternProperties": {
"^.+$": {"type": ["string", "number", "null"]}
}
}
},
"additionalProperties": false,
"patternProperties": {"^x-": {}}
},
"mac_address": {"type": "string"},
"mem_limit": {"type": ["number", "string"]},
"mem_reservation": {"type": ["string", "integer"]},
"mem_swappiness": {"type": "integer"},
"memswap_limit": {"type": ["number", "string"]},
"network_mode": {"type": "string"},
"networks": {
"oneOf": [
{"$ref": "#/definitions/list_of_strings"},
{
"type": "object",
"patternProperties": {
"^[a-zA-Z0-9._-]+$": {
"oneOf": [
{
"type": "object",
"properties": {
"aliases": {"$ref": "#/definitions/list_of_strings"},
"ipv4_address": {"type": "string"},
"ipv6_address": {"type": "string"},
"link_local_ips": {"$ref": "#/definitions/list_of_strings"},
"priority": {"type": "number"}
},
"additionalProperties": false,
"patternProperties": {"^x-": {}}
},
{"type": "null"}
]
}
},
"additionalProperties": false
}
]
},
"oom_kill_disable": {"type": "boolean"},
"oom_score_adj": {"type": "integer", "minimum": -1000, "maximum": 1000},
"pid": {"type": ["string", "null"]},
"pids_limit": {"type": ["number", "string"]},
"platform": {"type": "string"},
"ports": {
"type": "array",
"items": {
"oneOf": [
{"type": "number", "format": "ports"},
{"type": "string", "format": "ports"},
{
"type": "object",
"properties": {
"mode": {"type": "string"},
"target": {"type": "integer"},
"published": {"type": "integer"},
"protocol": {"type": "string"}
},
"additionalProperties": false,
"patternProperties": {"^x-": {}}
}
]
},
"uniqueItems": true
},
"privileged": {"type": "boolean"},
"profiles": {"$ref": "#/definitions/list_of_strings"},
"pull_policy": {"type": "string", "enum": [
"always", "never", "if_not_present", "build"
]},
"read_only": {"type": "boolean"},
"restart": {"type": "string"},
"runtime": {
"type": "string"
},
"scale": {
"type": "integer"
},
"security_opt": {"type": "array", "items": {"type": "string"}, "uniqueItems": true},
"shm_size": {"type": ["number", "string"]},
"secrets": {
"type": "array",
"items": {
"oneOf": [
{"type": "string"},
{
"type": "object",
"properties": {
"source": {"type": "string"},
"target": {"type": "string"},
"uid": {"type": "string"},
"gid": {"type": "string"},
"mode": {"type": "number"}
},
"additionalProperties": false,
"patternProperties": {"^x-": {}}
}
]
}
},
"sysctls": {"$ref": "#/definitions/list_or_dict"},
"stdin_open": {"type": "boolean"},
"stop_grace_period": {"type": "string", "format": "duration"},
"stop_signal": {"type": "string"},
"storage_opt": {"type": "object"},
"tmpfs": {"$ref": "#/definitions/string_or_list"},
"tty": {"type": "boolean"},
"ulimits": {
"type": "object",
"patternProperties": {
"^[a-z]+$": {
"oneOf": [
{"type": "integer"},
{
"type": "object",
"properties": {
"hard": {"type": "integer"},
"soft": {"type": "integer"}
},
"required": ["soft", "hard"],
"additionalProperties": false,
"patternProperties": {"^x-": {}}
}
]
}
}
},
"user": {"type": "string"},
"userns_mode": {"type": "string"},
"volumes": {
"type": "array",
"items": {
"oneOf": [
{"type": "string"},
{
"type": "object",
"required": ["type"],
"properties": {
"type": {"type": "string"},
"source": {"type": "string"},
"target": {"type": "string"},
"read_only": {"type": "boolean"},
"consistency": {"type": "string"},
"bind": {
"type": "object",
"properties": {
"propagation": {"type": "string"}
},
"additionalProperties": false,
"patternProperties": {"^x-": {}}
},
"volume": {
"type": "object",
"properties": {
"nocopy": {"type": "boolean"}
},
"additionalProperties": false,
"patternProperties": {"^x-": {}}
},
"tmpfs": {
"type": "object",
"properties": {
"size": {
"type": "integer",
"minimum": 0
}
},
"additionalProperties": false,
"patternProperties": {"^x-": {}}
}
},
"additionalProperties": false,
"patternProperties": {"^x-": {}}
}
]
},
"uniqueItems": true
},
"volumes_from": {
"type": "array",
"items": {"type": "string"},
"uniqueItems": true
},
"working_dir": {"type": "string"}
},
"patternProperties": {"^x-": {}},
"additionalProperties": false
},
"healthcheck": {
"id": "#/definitions/healthcheck",
"type": "object",
"properties": {
"disable": {"type": "boolean"},
"interval": {"type": "string", "format": "duration"},
"retries": {"type": "number"},
"test": {
"oneOf": [
{"type": "string"},
{"type": "array", "items": {"type": "string"}}
]
},
"timeout": {"type": "string", "format": "duration"},
"start_period": {"type": "string", "format": "duration"}
},
"additionalProperties": false,
"patternProperties": {"^x-": {}}
},
"deployment": {
"id": "#/definitions/deployment",
"type": ["object", "null"],
"properties": {
"mode": {"type": "string"},
"endpoint_mode": {"type": "string"},
"replicas": {"type": "integer"},
"labels": {"$ref": "#/definitions/list_or_dict"},
"rollback_config": {
"type": "object",
"properties": {
"parallelism": {"type": "integer"},
"delay": {"type": "string", "format": "duration"},
"failure_action": {"type": "string"},
"monitor": {"type": "string", "format": "duration"},
"max_failure_ratio": {"type": "number"},
"order": {"type": "string", "enum": [
"start-first", "stop-first"
]}
},
"additionalProperties": false,
"patternProperties": {"^x-": {}}
},
"update_config": {
"type": "object",
"properties": {
"parallelism": {"type": "integer"},
"delay": {"type": "string", "format": "duration"},
"failure_action": {"type": "string"},
"monitor": {"type": "string", "format": "duration"},
"max_failure_ratio": {"type": "number"},
"order": {"type": "string", "enum": [
"start-first", "stop-first"
]}
},
"additionalProperties": false,
"patternProperties": {"^x-": {}}
},
"resources": {
"type": "object",
"properties": {
"limits": {
"type": "object",
"properties": {
"cpus": {"type": ["number", "string"]},
"memory": {"type": "string"}
},
"additionalProperties": false,
"patternProperties": {"^x-": {}}
},
"reservations": {
"type": "object",
"properties": {
"cpus": {"type": ["number", "string"]},
"memory": {"type": "string"},
"generic_resources": {"$ref": "#/definitions/generic_resources"},
"devices": {"$ref": "#/definitions/devices"}
},
"additionalProperties": false,
"patternProperties": {"^x-": {}}
}
},
"additionalProperties": false,
"patternProperties": {"^x-": {}}
},
"restart_policy": {
"type": "object",
"properties": {
"condition": {"type": "string"},
"delay": {"type": "string", "format": "duration"},
"max_attempts": {"type": "integer"},
"window": {"type": "string", "format": "duration"}
},
"additionalProperties": false,
"patternProperties": {"^x-": {}}
},
"placement": {
"type": "object",
"properties": {
"constraints": {"type": "array", "items": {"type": "string"}},
"preferences": {
"type": "array",
"items": {
"type": "object",
"properties": {
"spread": {"type": "string"}
},
"additionalProperties": false,
"patternProperties": {"^x-": {}}
}
},
"max_replicas_per_node": {"type": "integer"}
},
"additionalProperties": false,
"patternProperties": {"^x-": {}}
}
},
"additionalProperties": false,
"patternProperties": {"^x-": {}}
},
"generic_resources": {
"id": "#/definitions/generic_resources",
"type": "array",
"items": {
"type": "object",
"properties": {
"discrete_resource_spec": {
"type": "object",
"properties": {
"kind": {"type": "string"},
"value": {"type": "number"}
},
"additionalProperties": false,
"patternProperties": {"^x-": {}}
}
},
"additionalProperties": false,
"patternProperties": {"^x-": {}}
}
},
"devices": {
"id": "#/definitions/devices",
"type": "array",
"items": {
"type": "object",
"properties": {
"capabilities": {"$ref": "#/definitions/list_of_strings"},
"count": {"type": ["string", "integer"]},
"device_ids": {"$ref": "#/definitions/list_of_strings"},
"driver":{"type": "string"},
"options":{"$ref": "#/definitions/list_or_dict"}
},
"additionalProperties": false,
"patternProperties": {"^x-": {}}
}
},
"network": {
"id": "#/definitions/network",
"type": ["object", "null"],
"properties": {
"name": {"type": "string"},
"driver": {"type": "string"},
"driver_opts": {
"type": "object",
"patternProperties": {
"^.+$": {"type": ["string", "number"]}
}
},
"ipam": {
"type": "object",
"properties": {
"driver": {"type": "string"},
"config": {
"type": "array",
"items": {
"type": "object",
"properties": {
"subnet": {"type": "string", "format": "subnet_ip_address"},
"ip_range": {"type": "string"},
"gateway": {"type": "string"},
"aux_addresses": {
"type": "object",
"additionalProperties": false,
"patternProperties": {"^.+$": {"type": "string"}}
}
},
"additionalProperties": false,
"patternProperties": {"^x-": {}}
}
},
"options": {
"type": "object",
"additionalProperties": false,
"patternProperties": {"^.+$": {"type": "string"}}
}
},
"additionalProperties": false,
"patternProperties": {"^x-": {}}
},
"external": {
"type": ["boolean", "object"],
"properties": {
"name": {
"deprecated": true,
"type": "string"
}
},
"additionalProperties": false,
"patternProperties": {"^x-": {}}
},
"internal": {"type": "boolean"},
"enable_ipv6": {"type": "boolean"},
"attachable": {"type": "boolean"},
"labels": {"$ref": "#/definitions/list_or_dict"}
},
"additionalProperties": false,
"patternProperties": {"^x-": {}}
},
"volume": {
"id": "#/definitions/volume",
"type": ["object", "null"],
"properties": {
"name": {"type": "string"},
"driver": {"type": "string"},
"driver_opts": {
"type": "object",
"patternProperties": {
"^.+$": {"type": ["string", "number"]}
}
},
"external": {
"type": ["boolean", "object"],
"properties": {
"name": {
"deprecated": true,
"type": "string"
}
},
"additionalProperties": false,
"patternProperties": {"^x-": {}}
},
"labels": {"$ref": "#/definitions/list_or_dict"}
},
"additionalProperties": false,
"patternProperties": {"^x-": {}}
},
"secret": {
"id": "#/definitions/secret",
"type": "object",
"properties": {
"name": {"type": "string"},
"file": {"type": "string"},
"external": {
"type": ["boolean", "object"],
"properties": {
"name": {"type": "string"}
}
},
"labels": {"$ref": "#/definitions/list_or_dict"},
"driver": {"type": "string"},
"driver_opts": {
"type": "object",
"patternProperties": {
"^.+$": {"type": ["string", "number"]}
}
},
"template_driver": {"type": "string"}
},
"additionalProperties": false,
"patternProperties": {"^x-": {}}
},
"config": {
"id": "#/definitions/config",
"type": "object",
"properties": {
"name": {"type": "string"},
"file": {"type": "string"},
"external": {
"type": ["boolean", "object"],
"properties": {
"name": {
"deprecated": true,
"type": "string"
}
}
},
"labels": {"$ref": "#/definitions/list_or_dict"},
"template_driver": {"type": "string"}
},
"additionalProperties": false,
"patternProperties": {"^x-": {}}
},
"string_or_list": {
"oneOf": [
{"type": "string"},
{"$ref": "#/definitions/list_of_strings"}
]
},
"list_of_strings": {
"type": "array",
"items": {"type": "string"},
"uniqueItems": true
},
"list_or_dict": {
"oneOf": [
{
"type": "object",
"patternProperties": {
".+": {
"type": ["string", "number", "null"]
}
},
"additionalProperties": false
},
{"type": "array", "items": {"type": "string"}, "uniqueItems": true}
]
},
"blkio_limit": {
"type": "object",
"properties": {
"path": {"type": "string"},
"rate": {"type": ["integer", "string"]}
},
"additionalProperties": false
},
"blkio_weight": {
"type": "object",
"properties": {
"path": {"type": "string"},
"weight": {"type": "integer"}
},
"additionalProperties": false
},
"constraints": {
"service": {
"id": "#/definitions/constraints/service",
"anyOf": [
{"required": ["build"]},
{"required": ["image"]}
],
"properties": {
"build": {
"required": ["context"]
}
}
}
}
}
}

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,203 @@
{
"$schema": "http://json-schema.org/draft-04/schema#",
"id": "config_schema_v1.json",
"type": "object",
"patternProperties": {
"^[a-zA-Z0-9._-]+$": {
"$ref": "#/definitions/service"
}
},
"additionalProperties": false,
"definitions": {
"service": {
"id": "#/definitions/service",
"type": "object",
"properties": {
"build": {"type": "string"},
"cap_add": {"type": "array", "items": {"type": "string"}, "uniqueItems": true},
"cap_drop": {"type": "array", "items": {"type": "string"}, "uniqueItems": true},
"cgroup_parent": {"type": "string"},
"command": {
"oneOf": [
{"type": "string"},
{"type": "array", "items": {"type": "string"}}
]
},
"container_name": {"type": "string"},
"cpu_shares": {"type": ["number", "string"]},
"cpu_quota": {"type": ["number", "string"]},
"cpuset": {"type": "string"},
"devices": {"type": "array", "items": {"type": "string"}, "uniqueItems": true},
"dns": {"$ref": "#/definitions/string_or_list"},
"dns_search": {"$ref": "#/definitions/string_or_list"},
"dockerfile": {"type": "string"},
"domainname": {"type": "string"},
"entrypoint": {
"oneOf": [
{"type": "string"},
{"type": "array", "items": {"type": "string"}}
]
},
"env_file": {"$ref": "#/definitions/string_or_list"},
"environment": {"$ref": "#/definitions/list_or_dict"},
"expose": {
"type": "array",
"items": {
"type": ["string", "number"],
"format": "expose"
},
"uniqueItems": true
},
"extends": {
"oneOf": [
{
"type": "string"
},
{
"type": "object",
"properties": {
"service": {"type": "string"},
"file": {"type": "string"}
},
"required": ["service"],
"additionalProperties": false
}
]
},
"extra_hosts": {"$ref": "#/definitions/list_or_dict"},
"external_links": {"type": "array", "items": {"type": "string"}, "uniqueItems": true},
"hostname": {"type": "string"},
"image": {"type": "string"},
"ipc": {"type": "string"},
"labels": {"$ref": "#/definitions/labels"},
"links": {"type": "array", "items": {"type": "string"}, "uniqueItems": true},
"log_driver": {"type": "string"},
"log_opt": {"type": "object"},
"mac_address": {"type": "string"},
"mem_limit": {"type": ["number", "string"]},
"memswap_limit": {"type": ["number", "string"]},
"mem_swappiness": {"type": "integer"},
"net": {"type": "string"},
"pid": {"type": ["string", "null"]},
"ports": {
"type": "array",
"items": {
"type": ["string", "number"],
"format": "ports"
},
"uniqueItems": true
},
"privileged": {"type": "boolean"},
"read_only": {"type": "boolean"},
"restart": {"type": "string"},
"security_opt": {"type": "array", "items": {"type": "string"}, "uniqueItems": true},
"shm_size": {"type": ["number", "string"]},
"stdin_open": {"type": "boolean"},
"stop_signal": {"type": "string"},
"tty": {"type": "boolean"},
"ulimits": {
"type": "object",
"patternProperties": {
"^[a-z]+$": {
"oneOf": [
{"type": "integer"},
{
"type":"object",
"properties": {
"hard": {"type": "integer"},
"soft": {"type": "integer"}
},
"required": ["soft", "hard"],
"additionalProperties": false
}
]
}
}
},
"user": {"type": "string"},
"volumes": {"type": "array", "items": {"type": "string"}, "uniqueItems": true},
"volume_driver": {"type": "string"},
"volumes_from": {"type": "array", "items": {"type": "string"}, "uniqueItems": true},
"working_dir": {"type": "string"}
},
"dependencies": {
"memswap_limit": ["mem_limit"]
},
"additionalProperties": false
},
"string_or_list": {
"oneOf": [
{"type": "string"},
{"$ref": "#/definitions/list_of_strings"}
]
},
"list_of_strings": {
"type": "array",
"items": {"type": "string"},
"uniqueItems": true
},
"list_or_dict": {
"oneOf": [
{
"type": "object",
"patternProperties": {
".+": {
"type": ["string", "number", "null"]
}
},
"additionalProperties": false
},
{"type": "array", "items": {"type": "string"}, "uniqueItems": true}
]
},
"labels": {
"oneOf": [
{
"type": "object",
"patternProperties": {
".+": {
"type": "string"
}
},
"additionalProperties": false
},
{"type": "array", "items": {"type": "string"}, "uniqueItems": true}
]
},
"constraints": {
"service": {
"id": "#/definitions/constraints/service",
"anyOf": [
{
"required": ["build"],
"not": {"required": ["image"]}
},
{
"required": ["image"],
"not": {"anyOf": [
{"required": ["build"]},
{"required": ["dockerfile"]}
]}
}
]
}
}
}
}

View File

@@ -0,0 +1,126 @@
import logging
import os
import re
import dotenv
from ..const import IS_WINDOWS_PLATFORM
from .errors import ConfigurationError
from .errors import EnvFileNotFound
log = logging.getLogger(__name__)
def split_env(env):
if isinstance(env, bytes):
env = env.decode('utf-8', 'replace')
key = value = None
if '=' in env:
key, value = env.split('=', 1)
else:
key = env
if re.search(r'\s', key):
raise ConfigurationError(
"environment variable name '{}' may not contain whitespace.".format(key)
)
return key, value
def env_vars_from_file(filename, interpolate=True):
"""
Read in a line delimited file of environment variables.
"""
if not os.path.exists(filename):
raise EnvFileNotFound("Couldn't find env file: {}".format(filename))
elif not os.path.isfile(filename):
raise EnvFileNotFound("{} is not a file.".format(filename))
env = dotenv.dotenv_values(dotenv_path=filename, encoding='utf-8-sig', interpolate=interpolate)
for k, v in env.items():
env[k] = v if interpolate else v.replace('$', '$$')
return env
class Environment(dict):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.missing_keys = []
self.silent = False
@classmethod
def from_env_file(cls, base_dir, env_file=None):
def _initialize():
result = cls()
if base_dir is None:
return result
if env_file:
env_file_path = os.path.join(os.getcwd(), env_file)
return cls(env_vars_from_file(env_file_path))
env_file_path = os.path.join(base_dir, '.env')
try:
return cls(env_vars_from_file(env_file_path))
except EnvFileNotFound:
pass
return result
instance = _initialize()
instance.update(os.environ)
return instance
@classmethod
def from_command_line(cls, parsed_env_opts):
result = cls()
for k, v in parsed_env_opts.items():
# Values from the command line take priority, unless they're unset
# in which case they take the value from the system's environment
if v is None and k in os.environ:
result[k] = os.environ[k]
else:
result[k] = v
return result
def __getitem__(self, key):
try:
return super().__getitem__(key)
except KeyError:
if IS_WINDOWS_PLATFORM:
try:
return super().__getitem__(key.upper())
except KeyError:
pass
if not self.silent and key not in self.missing_keys:
log.warning(
"The {} variable is not set. Defaulting to a blank string."
.format(key)
)
self.missing_keys.append(key)
return ""
def __contains__(self, key):
result = super().__contains__(key)
if IS_WINDOWS_PLATFORM:
return (
result or super().__contains__(key.upper())
)
return result
def get(self, key, *args, **kwargs):
if IS_WINDOWS_PLATFORM:
return super().get(
key,
super().get(key.upper(), *args, **kwargs)
)
return super().get(key, *args, **kwargs)
def get_boolean(self, key, default=False):
# Convert a value to a boolean using "common sense" rules.
# Unset, empty, "0" and "false" (i-case) yield False.
# All other values yield True.
value = self.get(key)
if not value:
return default
if value.lower() in ['0', 'false']:
return False
return True

View File

@@ -0,0 +1,55 @@
VERSION_EXPLANATION = (
'You might be seeing this error because you\'re using the wrong Compose file version. '
'Either specify a supported version (e.g "2.2" or "3.3") and place '
'your service definitions under the `services` key, or omit the `version` key '
'and place your service definitions at the root of the file to use '
'version 1.\nFor more on the Compose file format versions, see '
'https://docs.docker.com/compose/compose-file/')
class ConfigurationError(Exception):
def __init__(self, msg):
self.msg = msg
def __str__(self):
return self.msg
class EnvFileNotFound(ConfigurationError):
pass
class DependencyError(ConfigurationError):
pass
class CircularReference(ConfigurationError):
def __init__(self, trail):
self.trail = trail
@property
def msg(self):
lines = [
"{} in {}".format(service_name, filename)
for (filename, service_name) in self.trail
]
return "Circular reference:\n {}".format("\n extends ".join(lines))
class ComposeFileNotFound(ConfigurationError):
def __init__(self, supported_filenames):
super().__init__("""
Can't find a suitable configuration file in this directory or any
parent. Are you in the right directory?
Supported filenames: %s
""" % ", ".join(supported_filenames))
class DuplicateOverrideFileFound(ConfigurationError):
def __init__(self, override_filenames):
self.override_filenames = override_filenames
super().__init__(
"Multiple override files found: {}. You may only use a single "
"override file.".format(", ".join(override_filenames))
)

View File

@@ -0,0 +1,296 @@
import logging
import re
from string import Template
from .errors import ConfigurationError
from compose.const import COMPOSEFILE_V1 as V1
from compose.utils import parse_bytes
from compose.utils import parse_nanoseconds_int
log = logging.getLogger(__name__)
class Interpolator:
def __init__(self, templater, mapping):
self.templater = templater
self.mapping = mapping
def interpolate(self, string):
try:
return self.templater(string).substitute(self.mapping)
except ValueError:
raise InvalidInterpolation(string)
def interpolate_environment_variables(version, config, section, environment):
if version == V1:
interpolator = Interpolator(Template, environment)
else:
interpolator = Interpolator(TemplateWithDefaults, environment)
def process_item(name, config_dict):
return {
key: interpolate_value(name, key, val, section, interpolator)
for key, val in (config_dict or {}).items()
}
return {
name: process_item(name, config_dict or {})
for name, config_dict in config.items()
}
def get_config_path(config_key, section, name):
return '{}/{}/{}'.format(section, name, config_key)
def interpolate_value(name, config_key, value, section, interpolator):
try:
return recursive_interpolate(value, interpolator, get_config_path(config_key, section, name))
except InvalidInterpolation as e:
raise ConfigurationError(
'Invalid interpolation format for "{config_key}" option '
'in {section} "{name}": "{string}"'.format(
config_key=config_key,
name=name,
section=section,
string=e.string))
except UnsetRequiredSubstitution as e:
raise ConfigurationError(
'Missing mandatory value for "{config_key}" option interpolating {value} '
'in {section} "{name}": {err}'.format(config_key=config_key,
value=value,
name=name,
section=section,
err=e.err)
)
def recursive_interpolate(obj, interpolator, config_path):
def append(config_path, key):
return '{}/{}'.format(config_path, key)
if isinstance(obj, str):
return converter.convert(config_path, interpolator.interpolate(obj))
if isinstance(obj, dict):
return {
key: recursive_interpolate(val, interpolator, append(config_path, key))
for key, val in obj.items()
}
if isinstance(obj, list):
return [recursive_interpolate(val, interpolator, config_path) for val in obj]
return converter.convert(config_path, obj)
class TemplateWithDefaults(Template):
pattern = r"""
{delim}(?:
(?P<escaped>{delim}) |
(?P<named>{id}) |
{{(?P<braced>{bid})}} |
(?P<invalid>)
)
""".format(
delim=re.escape('$'),
id=r'[_a-z][_a-z0-9]*',
bid=r'[_a-z][_a-z0-9]*(?:(?P<sep>:?[-?])[^}]*)?',
)
@staticmethod
def process_braced_group(braced, sep, mapping):
if ':-' == sep:
var, _, default = braced.partition(':-')
return mapping.get(var) or default
elif '-' == sep:
var, _, default = braced.partition('-')
return mapping.get(var, default)
elif ':?' == sep:
var, _, err = braced.partition(':?')
result = mapping.get(var)
if not result:
err = err or var
raise UnsetRequiredSubstitution(err)
return result
elif '?' == sep:
var, _, err = braced.partition('?')
if var in mapping:
return mapping.get(var)
err = err or var
raise UnsetRequiredSubstitution(err)
# Modified from python2.7/string.py
def substitute(self, mapping):
# Helper function for .sub()
def convert(mo):
named = mo.group('named') or mo.group('braced')
braced = mo.group('braced')
if braced is not None:
sep = mo.group('sep')
if sep:
return self.process_braced_group(braced, sep, mapping)
if named is not None:
val = mapping[named]
if isinstance(val, bytes):
val = val.decode('utf-8')
return '{}'.format(val)
if mo.group('escaped') is not None:
return self.delimiter
if mo.group('invalid') is not None:
self._invalid(mo)
raise ValueError('Unrecognized named group in pattern',
self.pattern)
return self.pattern.sub(convert, self.template)
class InvalidInterpolation(Exception):
def __init__(self, string):
self.string = string
class UnsetRequiredSubstitution(Exception):
def __init__(self, custom_err_msg):
self.err = custom_err_msg
PATH_JOKER = '[^/]+'
FULL_JOKER = '.+'
def re_path(*args):
return re.compile('^{}$'.format('/'.join(args)))
def re_path_basic(section, name):
return re_path(section, PATH_JOKER, name)
def service_path(*args):
return re_path('service', PATH_JOKER, *args)
def to_boolean(s):
if not isinstance(s, str):
return s
s = s.lower()
if s in ['y', 'yes', 'true', 'on']:
return True
elif s in ['n', 'no', 'false', 'off']:
return False
raise ValueError('"{}" is not a valid boolean value'.format(s))
def to_int(s):
if not isinstance(s, str):
return s
# We must be able to handle octal representation for `mode` values notably
if re.match('^0[0-9]+$', s.strip()):
s = '0o' + s[1:]
try:
return int(s, base=0)
except ValueError:
raise ValueError('"{}" is not a valid integer'.format(s))
def to_float(s):
if not isinstance(s, str):
return s
try:
return float(s)
except ValueError:
raise ValueError('"{}" is not a valid float'.format(s))
def to_str(o):
if isinstance(o, (bool, float, int)):
return '{}'.format(o)
return o
def bytes_to_int(s):
v = parse_bytes(s)
if v is None:
raise ValueError('"{}" is not a valid byte value'.format(s))
return v
def to_microseconds(v):
if not isinstance(v, str):
return v
return int(parse_nanoseconds_int(v) / 1000)
class ConversionMap:
map = {
service_path('blkio_config', 'weight'): to_int,
service_path('blkio_config', 'weight_device', 'weight'): to_int,
service_path('build', 'labels', FULL_JOKER): to_str,
service_path('cpus'): to_float,
service_path('cpu_count'): to_int,
service_path('cpu_quota'): to_microseconds,
service_path('cpu_period'): to_microseconds,
service_path('cpu_rt_period'): to_microseconds,
service_path('cpu_rt_runtime'): to_microseconds,
service_path('configs', 'mode'): to_int,
service_path('secrets', 'mode'): to_int,
service_path('healthcheck', 'retries'): to_int,
service_path('healthcheck', 'disable'): to_boolean,
service_path('deploy', 'labels', PATH_JOKER): to_str,
service_path('deploy', 'replicas'): to_int,
service_path('deploy', 'placement', 'max_replicas_per_node'): to_int,
service_path('deploy', 'resources', 'limits', "cpus"): to_float,
service_path('deploy', 'update_config', 'parallelism'): to_int,
service_path('deploy', 'update_config', 'max_failure_ratio'): to_float,
service_path('deploy', 'rollback_config', 'parallelism'): to_int,
service_path('deploy', 'rollback_config', 'max_failure_ratio'): to_float,
service_path('deploy', 'restart_policy', 'max_attempts'): to_int,
service_path('mem_swappiness'): to_int,
service_path('labels', FULL_JOKER): to_str,
service_path('oom_kill_disable'): to_boolean,
service_path('oom_score_adj'): to_int,
service_path('ports', 'target'): to_int,
service_path('ports', 'published'): to_int,
service_path('scale'): to_int,
service_path('ulimits', PATH_JOKER): to_int,
service_path('ulimits', PATH_JOKER, 'soft'): to_int,
service_path('ulimits', PATH_JOKER, 'hard'): to_int,
service_path('privileged'): to_boolean,
service_path('read_only'): to_boolean,
service_path('stdin_open'): to_boolean,
service_path('tty'): to_boolean,
service_path('volumes', 'read_only'): to_boolean,
service_path('volumes', 'volume', 'nocopy'): to_boolean,
service_path('volumes', 'tmpfs', 'size'): bytes_to_int,
re_path_basic('network', 'attachable'): to_boolean,
re_path_basic('network', 'external'): to_boolean,
re_path_basic('network', 'internal'): to_boolean,
re_path('network', PATH_JOKER, 'labels', FULL_JOKER): to_str,
re_path_basic('volume', 'external'): to_boolean,
re_path('volume', PATH_JOKER, 'labels', FULL_JOKER): to_str,
re_path_basic('secret', 'external'): to_boolean,
re_path('secret', PATH_JOKER, 'labels', FULL_JOKER): to_str,
re_path_basic('config', 'external'): to_boolean,
re_path('config', PATH_JOKER, 'labels', FULL_JOKER): to_str,
}
def convert(self, path, value):
for rexp in self.map.keys():
if rexp.match(path):
try:
return self.map[rexp](value)
except ValueError as e:
raise ConfigurationError(
'Error while attempting to convert {} to appropriate type: {}'.format(
path.replace('/', '.'), e
)
)
return value
converter = ConversionMap()

View File

@@ -0,0 +1,149 @@
import yaml
from compose.config import types
from compose.const import COMPOSE_SPEC as VERSION
from compose.const import COMPOSEFILE_V1 as V1
def serialize_config_type(dumper, data):
representer = dumper.represent_str
return representer(data.repr())
def serialize_dict_type(dumper, data):
return dumper.represent_dict(data.repr())
def serialize_string(dumper, data):
""" Ensure boolean-like strings are quoted in the output """
representer = dumper.represent_str
if isinstance(data, bytes):
data = data.decode('utf-8')
if data.lower() in ('y', 'n', 'yes', 'no', 'on', 'off', 'true', 'false'):
# Empirically only y/n appears to be an issue, but this might change
# depending on which PyYaml version is being used. Err on safe side.
return dumper.represent_scalar('tag:yaml.org,2002:str', data, style='"')
return representer(data)
def serialize_string_escape_dollar(dumper, data):
""" Ensure boolean-like strings are quoted in the output and escape $ characters """
data = data.replace('$', '$$')
return serialize_string(dumper, data)
yaml.SafeDumper.add_representer(types.MountSpec, serialize_dict_type)
yaml.SafeDumper.add_representer(types.VolumeFromSpec, serialize_config_type)
yaml.SafeDumper.add_representer(types.VolumeSpec, serialize_config_type)
yaml.SafeDumper.add_representer(types.SecurityOpt, serialize_config_type)
yaml.SafeDumper.add_representer(types.ServiceSecret, serialize_dict_type)
yaml.SafeDumper.add_representer(types.ServiceConfig, serialize_dict_type)
yaml.SafeDumper.add_representer(types.ServicePort, serialize_dict_type)
def denormalize_config(config, image_digests=None):
result = {'version': str(config.config_version)}
denormalized_services = [
denormalize_service_dict(
service_dict,
config.version,
image_digests[service_dict['name']] if image_digests else None)
for service_dict in config.services
]
result['services'] = {
service_dict.pop('name'): service_dict
for service_dict in denormalized_services
}
for key in ('networks', 'volumes', 'secrets', 'configs'):
config_dict = getattr(config, key)
if not config_dict:
continue
result[key] = config_dict.copy()
for name, conf in result[key].items():
if 'external_name' in conf:
del conf['external_name']
if 'name' in conf:
if 'external' in conf:
conf['external'] = bool(conf['external'])
return result
def serialize_config(config, image_digests=None, escape_dollar=True):
if escape_dollar:
yaml.SafeDumper.add_representer(str, serialize_string_escape_dollar)
yaml.SafeDumper.add_representer(str, serialize_string_escape_dollar)
else:
yaml.SafeDumper.add_representer(str, serialize_string)
yaml.SafeDumper.add_representer(str, serialize_string)
return yaml.safe_dump(
denormalize_config(config, image_digests),
default_flow_style=False,
indent=2,
width=80,
allow_unicode=True
)
def serialize_ns_time_value(value):
result = (value, 'ns')
table = [
(1000., 'us'),
(1000., 'ms'),
(1000., 's'),
(60., 'm'),
(60., 'h')
]
for stage in table:
tmp = value / stage[0]
if tmp == int(value / stage[0]):
value = tmp
result = (int(value), stage[1])
else:
break
return '{}{}'.format(*result)
def denormalize_service_dict(service_dict, version, image_digest=None):
service_dict = service_dict.copy()
if image_digest:
service_dict['image'] = image_digest
if 'restart' in service_dict:
service_dict['restart'] = types.serialize_restart_spec(
service_dict['restart']
)
if version == V1 and 'network_mode' not in service_dict:
service_dict['network_mode'] = 'bridge'
if 'healthcheck' in service_dict:
if 'interval' in service_dict['healthcheck']:
service_dict['healthcheck']['interval'] = serialize_ns_time_value(
service_dict['healthcheck']['interval']
)
if 'timeout' in service_dict['healthcheck']:
service_dict['healthcheck']['timeout'] = serialize_ns_time_value(
service_dict['healthcheck']['timeout']
)
if 'start_period' in service_dict['healthcheck']:
service_dict['healthcheck']['start_period'] = serialize_ns_time_value(
service_dict['healthcheck']['start_period']
)
if 'ports' in service_dict:
service_dict['ports'] = [
p.legacy_repr() if p.external_ip or version < VERSION else p
for p in service_dict['ports']
]
if 'volumes' in service_dict and (version == V1):
service_dict['volumes'] = [
v.legacy_repr() if isinstance(v, types.MountSpec) else v for v in service_dict['volumes']
]
return service_dict

View File

@@ -0,0 +1,71 @@
from compose.config.errors import DependencyError
def get_service_name_from_network_mode(network_mode):
return get_source_name_from_network_mode(network_mode, 'service')
def get_container_name_from_network_mode(network_mode):
return get_source_name_from_network_mode(network_mode, 'container')
def get_source_name_from_network_mode(network_mode, source_type):
if not network_mode:
return
if not network_mode.startswith(source_type+':'):
return
_, net_name = network_mode.split(':', 1)
return net_name
def get_service_names(links):
return [link.split(':', 1)[0] for link in links]
def get_service_names_from_volumes_from(volumes_from):
return [volume_from.source for volume_from in volumes_from]
def get_service_dependents(service_dict, services):
name = service_dict['name']
return [
service for service in services
if (name in get_service_names(service.get('links', [])) or
name in get_service_names_from_volumes_from(service.get('volumes_from', [])) or
name == get_service_name_from_network_mode(service.get('network_mode')) or
name == get_service_name_from_network_mode(service.get('pid')) or
name == get_service_name_from_network_mode(service.get('ipc')) or
name in service.get('depends_on', []))
]
def sort_service_dicts(services):
# Topological sort (Cormen/Tarjan algorithm).
unmarked = services[:]
temporary_marked = set()
sorted_services = []
def visit(n):
if n['name'] in temporary_marked:
if n['name'] in get_service_names(n.get('links', [])):
raise DependencyError('A service can not link to itself: %s' % n['name'])
if n['name'] in n.get('volumes_from', []):
raise DependencyError('A service can not mount itself as volume: %s' % n['name'])
if n['name'] in n.get('depends_on', []):
raise DependencyError('A service can not depend on itself: %s' % n['name'])
raise DependencyError('Circular dependency between %s' % ' and '.join(temporary_marked))
if n in unmarked:
temporary_marked.add(n['name'])
for m in get_service_dependents(n, services):
visit(m)
temporary_marked.remove(n['name'])
unmarked.remove(n)
sorted_services.insert(0, n)
while unmarked:
visit(unmarked[-1])
return sorted_services

View File

@@ -0,0 +1,500 @@
"""
Types for objects parsed from the configuration.
"""
import json
import ntpath
import os
import re
from collections import namedtuple
from docker.utils.ports import build_port_bindings
from ..const import COMPOSEFILE_V1 as V1
from ..utils import unquote_path
from .errors import ConfigurationError
from compose.const import IS_WINDOWS_PLATFORM
from compose.utils import splitdrive
win32_root_path_pattern = re.compile(r'^[A-Za-z]\:\\.*')
class VolumeFromSpec(namedtuple('_VolumeFromSpec', 'source mode type')):
# TODO: drop service_names arg when v1 is removed
@classmethod
def parse(cls, volume_from_config, service_names, version):
func = cls.parse_v1 if version == V1 else cls.parse_v2
return func(service_names, volume_from_config)
@classmethod
def parse_v1(cls, service_names, volume_from_config):
parts = volume_from_config.split(':')
if len(parts) > 2:
raise ConfigurationError(
"volume_from {} has incorrect format, should be "
"service[:mode]".format(volume_from_config))
if len(parts) == 1:
source = parts[0]
mode = 'rw'
else:
source, mode = parts
type = 'service' if source in service_names else 'container'
return cls(source, mode, type)
@classmethod
def parse_v2(cls, service_names, volume_from_config):
parts = volume_from_config.split(':')
if len(parts) > 3:
raise ConfigurationError(
"volume_from {} has incorrect format, should be one of "
"'<service name>[:<mode>]' or "
"'container:<container name>[:<mode>]'".format(volume_from_config))
if len(parts) == 1:
source = parts[0]
return cls(source, 'rw', 'service')
if len(parts) == 2:
if parts[0] == 'container':
type, source = parts
return cls(source, 'rw', type)
source, mode = parts
return cls(source, mode, 'service')
if len(parts) == 3:
type, source, mode = parts
if type not in ('service', 'container'):
raise ConfigurationError(
"Unknown volumes_from type '{}' in '{}'".format(
type,
volume_from_config))
return cls(source, mode, type)
def repr(self):
return '{v.type}:{v.source}:{v.mode}'.format(v=self)
def parse_restart_spec(restart_config):
if not restart_config:
return None
parts = restart_config.split(':')
if len(parts) > 2:
raise ConfigurationError(
"Restart %s has incorrect format, should be "
"mode[:max_retry]" % restart_config)
if len(parts) == 2:
name, max_retry_count = parts
else:
name, = parts
max_retry_count = 0
return {'Name': name, 'MaximumRetryCount': int(max_retry_count)}
def serialize_restart_spec(restart_spec):
if not restart_spec:
return ''
parts = [restart_spec['Name']]
if restart_spec['MaximumRetryCount']:
parts.append(str(restart_spec['MaximumRetryCount']))
return ':'.join(parts)
def parse_extra_hosts(extra_hosts_config):
if not extra_hosts_config:
return {}
if isinstance(extra_hosts_config, dict):
return dict(extra_hosts_config)
if isinstance(extra_hosts_config, list):
extra_hosts_dict = {}
for extra_hosts_line in extra_hosts_config:
# TODO: validate string contains ':' ?
host, ip = extra_hosts_line.split(':', 1)
extra_hosts_dict[host.strip()] = ip.strip()
return extra_hosts_dict
def normalize_path_for_engine(path):
"""Windows paths, c:\\my\\path\\shiny, need to be changed to be compatible with
the Engine. Volume paths are expected to be linux style /c/my/path/shiny/
"""
drive, tail = splitdrive(path)
if drive:
path = '/' + drive.lower().rstrip(':') + tail
return path.replace('\\', '/')
def normpath(path, win_host=False):
""" Custom path normalizer that handles Compose-specific edge cases like
UNIX paths on Windows hosts and vice-versa. """
sysnorm = ntpath.normpath if win_host else os.path.normpath
# If a path looks like a UNIX absolute path on Windows, it probably is;
# we'll need to revert the backslashes to forward slashes after normalization
flip_slashes = path.startswith('/') and IS_WINDOWS_PLATFORM
path = sysnorm(path)
if flip_slashes:
path = path.replace('\\', '/')
return path
class MountSpec:
options_map = {
'volume': {
'nocopy': 'no_copy'
},
'bind': {
'propagation': 'propagation'
},
'tmpfs': {
'size': 'tmpfs_size'
}
}
_fields = ['type', 'source', 'target', 'read_only', 'consistency']
@classmethod
def parse(cls, mount_dict, normalize=False, win_host=False):
if mount_dict.get('source'):
if mount_dict['type'] == 'tmpfs':
raise ConfigurationError('tmpfs mounts can not specify a source')
mount_dict['source'] = normpath(mount_dict['source'], win_host)
if normalize:
mount_dict['source'] = normalize_path_for_engine(mount_dict['source'])
return cls(**mount_dict)
def __init__(self, type, source=None, target=None, read_only=None, consistency=None, **kwargs):
self.type = type
self.source = source
self.target = target
self.read_only = read_only
self.consistency = consistency
self.options = None
if self.type in kwargs:
self.options = kwargs[self.type]
def as_volume_spec(self):
mode = 'ro' if self.read_only else 'rw'
return VolumeSpec(external=self.source, internal=self.target, mode=mode)
def legacy_repr(self):
return self.as_volume_spec().repr()
def repr(self):
res = {}
for field in self._fields:
if getattr(self, field, None):
res[field] = getattr(self, field)
if self.options:
res[self.type] = self.options
return res
@property
def is_named_volume(self):
return self.type == 'volume' and self.source
@property
def is_tmpfs(self):
return self.type == 'tmpfs'
@property
def external(self):
return self.source
class VolumeSpec(namedtuple('_VolumeSpec', 'external internal mode')):
win32 = False
@classmethod
def _parse_unix(cls, volume_config):
parts = volume_config.split(':')
if len(parts) > 3:
raise ConfigurationError(
"Volume %s has incorrect format, should be "
"external:internal[:mode]" % volume_config)
if len(parts) == 1:
external = None
internal = os.path.normpath(parts[0])
else:
external = os.path.normpath(parts[0])
internal = os.path.normpath(parts[1])
mode = 'rw'
if len(parts) == 3:
mode = parts[2]
return cls(external, internal, mode)
@classmethod
def _parse_win32(cls, volume_config, normalize):
# relative paths in windows expand to include the drive, eg C:\
# so we join the first 2 parts back together to count as one
mode = 'rw'
def separate_next_section(volume_config):
drive, tail = splitdrive(volume_config)
parts = tail.split(':', 1)
if drive:
parts[0] = drive + parts[0]
return parts
parts = separate_next_section(volume_config)
if len(parts) == 1:
internal = parts[0]
external = None
else:
external = parts[0]
parts = separate_next_section(parts[1])
external = normpath(external, True)
internal = parts[0]
if len(parts) > 1:
if ':' in parts[1]:
raise ConfigurationError(
"Volume %s has incorrect format, should be "
"external:internal[:mode]" % volume_config
)
mode = parts[1]
if normalize:
external = normalize_path_for_engine(external) if external else None
result = cls(external, internal, mode)
result.win32 = True
return result
@classmethod
def parse(cls, volume_config, normalize=False, win_host=False):
"""Parse a volume_config path and split it into external:internal[:mode]
parts to be returned as a valid VolumeSpec.
"""
if IS_WINDOWS_PLATFORM or win_host:
return cls._parse_win32(volume_config, normalize)
else:
return cls._parse_unix(volume_config)
def repr(self):
external = self.external + ':' if self.external else ''
mode = ':' + self.mode if self.external else ''
return '{ext}{v.internal}{mode}'.format(mode=mode, ext=external, v=self)
@property
def is_named_volume(self):
res = self.external and not self.external.startswith(('.', '/', '~'))
if not self.win32:
return res
return (
res and not self.external.startswith('\\') and
not win32_root_path_pattern.match(self.external)
)
class ServiceLink(namedtuple('_ServiceLink', 'target alias')):
@classmethod
def parse(cls, link_spec):
target, _, alias = link_spec.partition(':')
if not alias:
alias = target
return cls(target, alias)
def repr(self):
if self.target == self.alias:
return self.target
return '{s.target}:{s.alias}'.format(s=self)
@property
def merge_field(self):
return self.alias
class ServiceConfigBase(namedtuple('_ServiceConfigBase', 'source target uid gid mode name')):
@classmethod
def parse(cls, spec):
if isinstance(spec, str):
return cls(spec, None, None, None, None, None)
return cls(
spec.get('source'),
spec.get('target'),
spec.get('uid'),
spec.get('gid'),
spec.get('mode'),
spec.get('name')
)
@property
def merge_field(self):
return self.source
def repr(self):
return {
k: v for k, v in zip(self._fields, self) if v is not None
}
class ServiceSecret(ServiceConfigBase):
pass
class ServiceConfig(ServiceConfigBase):
pass
class ServicePort(namedtuple('_ServicePort', 'target published protocol mode external_ip')):
def __new__(cls, target, published, *args, **kwargs):
try:
if target:
target = int(target)
except ValueError:
raise ConfigurationError('Invalid target port: {}'.format(target))
if published:
if isinstance(published, str) and '-' in published: # "x-y:z" format
a, b = published.split('-', 1)
if not a.isdigit() or not b.isdigit():
raise ConfigurationError('Invalid published port: {}'.format(published))
else:
try:
published = int(published)
except ValueError:
raise ConfigurationError('Invalid published port: {}'.format(published))
return super().__new__(
cls, target, published, *args, **kwargs
)
@classmethod
def parse(cls, spec):
if isinstance(spec, cls):
# When extending a service with ports, the port definitions have already been parsed
return [spec]
if not isinstance(spec, dict):
result = []
try:
for k, v in build_port_bindings([spec]).items():
if '/' in k:
target, proto = k.split('/', 1)
else:
target, proto = (k, None)
for pub in v:
if pub is None:
result.append(
cls(target, None, proto, None, None)
)
elif isinstance(pub, tuple):
result.append(
cls(target, pub[1], proto, None, pub[0])
)
else:
result.append(
cls(target, pub, proto, None, None)
)
except ValueError as e:
raise ConfigurationError(str(e))
return result
return [cls(
spec.get('target'),
spec.get('published'),
spec.get('protocol'),
spec.get('mode'),
None
)]
@property
def merge_field(self):
return (self.target, self.published, self.external_ip, self.protocol)
def repr(self):
return {
k: v for k, v in zip(self._fields, self) if v is not None
}
def legacy_repr(self):
return normalize_port_dict(self.repr())
class GenericResource(namedtuple('_GenericResource', 'kind value')):
@classmethod
def parse(cls, dct):
if 'discrete_resource_spec' not in dct:
raise ConfigurationError(
'generic_resource entry must include a discrete_resource_spec key'
)
if 'kind' not in dct['discrete_resource_spec']:
raise ConfigurationError(
'generic_resource entry must include a discrete_resource_spec.kind subkey'
)
return cls(
dct['discrete_resource_spec']['kind'],
dct['discrete_resource_spec'].get('value')
)
def repr(self):
return {
'discrete_resource_spec': {
'kind': self.kind,
'value': self.value,
}
}
@property
def merge_field(self):
return self.kind
def normalize_port_dict(port):
return '{external_ip}{has_ext_ip}{published}{is_pub}{target}/{protocol}'.format(
published=port.get('published', ''),
is_pub=(':' if port.get('published') is not None or port.get('external_ip') else ''),
target=port.get('target'),
protocol=port.get('protocol', 'tcp'),
external_ip=port.get('external_ip', ''),
has_ext_ip=(':' if port.get('external_ip') else ''),
)
class SecurityOpt(namedtuple('_SecurityOpt', 'value src_file')):
@classmethod
def parse(cls, value):
if not isinstance(value, str):
return value
# based on https://github.com/docker/cli/blob/9de1b162f/cli/command/container/opts.go#L673-L697
con = value.split('=', 2)
if len(con) == 1 and con[0] != 'no-new-privileges':
if ':' not in value:
raise ConfigurationError('Invalid security_opt: {}'.format(value))
con = value.split(':', 2)
if con[0] == 'seccomp' and con[1] != 'unconfined':
try:
with open(unquote_path(con[1])) as f:
seccomp_data = json.load(f)
except (OSError, ValueError) as e:
raise ConfigurationError('Error reading seccomp profile: {}'.format(e))
return cls(
'seccomp={}'.format(json.dumps(seccomp_data)), con[1]
)
return cls(value, None)
def repr(self):
if self.src_file is not None:
return 'seccomp:{}'.format(self.src_file)
return self.value
@property
def merge_field(self):
return self.value

View File

@@ -0,0 +1,569 @@
import json
import logging
import os
import re
import sys
from docker.utils.ports import split_port
from jsonschema import Draft4Validator
from jsonschema import FormatChecker
from jsonschema import RefResolver
from jsonschema import ValidationError
from ..const import COMPOSEFILE_V1 as V1
from ..const import NANOCPUS_SCALE
from .errors import ConfigurationError
from .errors import VERSION_EXPLANATION
from .sort_services import get_service_name_from_network_mode
log = logging.getLogger(__name__)
DOCKER_CONFIG_HINTS = {
'cpu_share': 'cpu_shares',
'add_host': 'extra_hosts',
'hosts': 'extra_hosts',
'extra_host': 'extra_hosts',
'device': 'devices',
'link': 'links',
'memory_swap': 'memswap_limit',
'port': 'ports',
'privilege': 'privileged',
'priviliged': 'privileged',
'privilige': 'privileged',
'volume': 'volumes',
'workdir': 'working_dir',
}
VALID_NAME_CHARS = r'[a-zA-Z0-9\._\-]'
VALID_EXPOSE_FORMAT = r'^\d+(\-\d+)?(\/[a-zA-Z]+)?$'
VALID_IPV4_SEG = r'(\d{1,2}|1\d{2}|2[0-4]\d|25[0-5])'
VALID_IPV4_ADDR = r"({IPV4_SEG}\.){{3}}{IPV4_SEG}".format(IPV4_SEG=VALID_IPV4_SEG)
VALID_REGEX_IPV4_CIDR = r"^{IPV4_ADDR}/(\d|[1-2]\d|3[0-2])$".format(IPV4_ADDR=VALID_IPV4_ADDR)
VALID_IPV6_SEG = r'[0-9a-fA-F]{1,4}'
VALID_REGEX_IPV6_CIDR = "".join(r"""
^
(
(({IPV6_SEG}:){{7}}{IPV6_SEG})|
(({IPV6_SEG}:){{1,7}}:)|
(({IPV6_SEG}:){{1,6}}(:{IPV6_SEG}){{1,1}})|
(({IPV6_SEG}:){{1,5}}(:{IPV6_SEG}){{1,2}})|
(({IPV6_SEG}:){{1,4}}(:{IPV6_SEG}){{1,3}})|
(({IPV6_SEG}:){{1,3}}(:{IPV6_SEG}){{1,4}})|
(({IPV6_SEG}:){{1,2}}(:{IPV6_SEG}){{1,5}})|
(({IPV6_SEG}:){{1,1}}(:{IPV6_SEG}){{1,6}})|
(:((:{IPV6_SEG}){{1,7}}|:))|
(fe80:(:{IPV6_SEG}){{0,4}}%[0-9a-zA-Z]{{1,}})|
(::(ffff(:0{{1,4}}){{0,1}}:){{0,1}}{IPV4_ADDR})|
(({IPV6_SEG}:){{1,4}}:{IPV4_ADDR})
)
/(\d|[1-9]\d|1[0-1]\d|12[0-8])
$
""".format(IPV6_SEG=VALID_IPV6_SEG, IPV4_ADDR=VALID_IPV4_ADDR).split())
@FormatChecker.cls_checks(format="ports", raises=ValidationError)
def format_ports(instance):
try:
split_port(instance)
except ValueError as e:
raise ValidationError(str(e))
return True
@FormatChecker.cls_checks(format="expose", raises=ValidationError)
def format_expose(instance):
if isinstance(instance, str):
if not re.match(VALID_EXPOSE_FORMAT, instance):
raise ValidationError(
"should be of the format 'PORT[/PROTOCOL]'")
return True
@FormatChecker.cls_checks("subnet_ip_address", raises=ValidationError)
def format_subnet_ip_address(instance):
if isinstance(instance, str):
if not re.match(VALID_REGEX_IPV4_CIDR, instance) and \
not re.match(VALID_REGEX_IPV6_CIDR, instance):
raise ValidationError("should use the CIDR format")
return True
def match_named_volumes(service_dict, project_volumes):
service_volumes = service_dict.get('volumes', [])
for volume_spec in service_volumes:
if volume_spec.is_named_volume and volume_spec.external not in project_volumes:
raise ConfigurationError(
'Named volume "{}" is used in service "{}" but no'
' declaration was found in the volumes section.'.format(
volume_spec.repr(), service_dict.get('name')
)
)
def python_type_to_yaml_type(type_):
type_name = type(type_).__name__
return {
'dict': 'mapping',
'list': 'array',
'int': 'number',
'float': 'number',
'bool': 'boolean',
'unicode': 'string',
'str': 'string',
'bytes': 'string',
}.get(type_name, type_name)
def validate_config_section(filename, config, section):
"""Validate the structure of a configuration section. This must be done
before interpolation so it's separate from schema validation.
"""
if not isinstance(config, dict):
raise ConfigurationError(
"In file '{filename}', {section} must be a mapping, not "
"{type}.".format(
filename=filename,
section=section,
type=anglicize_json_type(python_type_to_yaml_type(config))))
for key, value in config.items():
if not isinstance(key, str):
raise ConfigurationError(
"In file '{filename}', the {section} name {name} must be a "
"quoted string, i.e. '{name}'.".format(
filename=filename,
section=section,
name=key))
if not isinstance(value, (dict, type(None))):
raise ConfigurationError(
"In file '{filename}', {section} '{name}' must be a mapping not "
"{type}.".format(
filename=filename,
section=section,
name=key,
type=anglicize_json_type(python_type_to_yaml_type(value))))
def validate_top_level_object(config_file):
if not isinstance(config_file.config, dict):
raise ConfigurationError(
"Top level object in '{}' needs to be an object not '{}'.".format(
config_file.filename,
type(config_file.config)))
def validate_ulimits(service_config):
ulimit_config = service_config.config.get('ulimits', {})
for limit_name, soft_hard_values in ulimit_config.items():
if isinstance(soft_hard_values, dict):
if not soft_hard_values['soft'] <= soft_hard_values['hard']:
raise ConfigurationError(
"Service '{s.name}' has invalid ulimit '{ulimit}'. "
"'soft' value can not be greater than 'hard' value ".format(
s=service_config,
ulimit=ulimit_config))
def validate_extends_file_path(service_name, extends_options, filename):
"""
The service to be extended must either be defined in the config key 'file',
or within 'filename'.
"""
error_prefix = "Invalid 'extends' configuration for %s:" % service_name
if 'file' not in extends_options and filename is None:
raise ConfigurationError(
"%s you need to specify a 'file', e.g. 'file: something.yml'" % error_prefix
)
def validate_network_mode(service_config, service_names):
network_mode = service_config.config.get('network_mode')
if not network_mode:
return
if 'networks' in service_config.config:
raise ConfigurationError("'network_mode' and 'networks' cannot be combined")
dependency = get_service_name_from_network_mode(network_mode)
if not dependency:
return
if dependency not in service_names:
raise ConfigurationError(
"Service '{s.name}' uses the network stack of service '{dep}' which "
"is undefined.".format(s=service_config, dep=dependency))
def validate_pid_mode(service_config, service_names):
pid_mode = service_config.config.get('pid')
if not pid_mode:
return
dependency = get_service_name_from_network_mode(pid_mode)
if not dependency:
return
if dependency not in service_names:
raise ConfigurationError(
"Service '{s.name}' uses the PID namespace of service '{dep}' which "
"is undefined.".format(s=service_config, dep=dependency)
)
def validate_ipc_mode(service_config, service_names):
ipc_mode = service_config.config.get('ipc')
if not ipc_mode:
return
dependency = get_service_name_from_network_mode(ipc_mode)
if not dependency:
return
if dependency not in service_names:
raise ConfigurationError(
"Service '{s.name}' uses the IPC namespace of service '{dep}' which "
"is undefined.".format(s=service_config, dep=dependency)
)
def validate_links(service_config, service_names):
for link in service_config.config.get('links', []):
if link.split(':')[0] not in service_names:
raise ConfigurationError(
"Service '{s.name}' has a link to service '{link}' which is "
"undefined.".format(s=service_config, link=link))
def validate_depends_on(service_config, service_names):
deps = service_config.config.get('depends_on', {})
for dependency in deps.keys():
if dependency not in service_names:
raise ConfigurationError(
"Service '{s.name}' depends on service '{dep}' which is "
"undefined.".format(s=service_config, dep=dependency)
)
def validate_credential_spec(service_config):
credential_spec = service_config.config.get('credential_spec')
if not credential_spec:
return
if 'registry' not in credential_spec and 'file' not in credential_spec:
raise ConfigurationError(
"Service '{s.name}' is missing 'credential_spec.file' or "
"credential_spec.registry'".format(s=service_config)
)
def get_unsupported_config_msg(path, error_key):
msg = "Unsupported config option for {}: '{}'".format(path_string(path), error_key)
if error_key in DOCKER_CONFIG_HINTS:
msg += " (did you mean '{}'?)".format(DOCKER_CONFIG_HINTS[error_key])
return msg
def anglicize_json_type(json_type):
if json_type.startswith(('a', 'e', 'i', 'o', 'u')):
return 'an ' + json_type
return 'a ' + json_type
def is_service_dict_schema(schema_id):
return schema_id in ('config_schema_v1.json', '#/properties/services')
def handle_error_for_schema_with_id(error, path):
schema_id = error.schema['id']
if is_service_dict_schema(schema_id) and error.validator == 'additionalProperties':
return "Invalid service name '{}' - only {} characters are allowed".format(
# The service_name is one of the keys in the json object
[i for i in list(error.instance) if not i or any(filter(
lambda c: not re.match(VALID_NAME_CHARS, c), i
))][0],
VALID_NAME_CHARS
)
if error.validator == 'additionalProperties':
if schema_id == '#/definitions/service':
invalid_config_key = parse_key_from_error_msg(error)
return get_unsupported_config_msg(path, invalid_config_key)
if schema_id.startswith('config_schema_'):
invalid_config_key = parse_key_from_error_msg(error)
return ('Invalid top-level property "{key}". Valid top-level '
'sections for this Compose file are: {properties}, and '
'extensions starting with "x-".\n\n{explanation}').format(
key=invalid_config_key,
properties=', '.join(error.schema['properties'].keys()),
explanation=VERSION_EXPLANATION
)
if not error.path:
return '{}\n\n{}'.format(error.message, VERSION_EXPLANATION)
def handle_generic_error(error, path):
msg_format = None
error_msg = error.message
if error.validator == 'oneOf':
msg_format = "{path} {msg}"
config_key, error_msg = _parse_oneof_validator(error)
if config_key:
path.append(config_key)
elif error.validator == 'type':
msg_format = "{path} contains an invalid type, it should be {msg}"
error_msg = _parse_valid_types_from_validator(error.validator_value)
elif error.validator == 'required':
error_msg = ", ".join(error.validator_value)
msg_format = "{path} is invalid, {msg} is required."
elif error.validator == 'dependencies':
config_key = list(error.validator_value.keys())[0]
required_keys = ",".join(error.validator_value[config_key])
msg_format = "{path} is invalid: {msg}"
path.append(config_key)
error_msg = "when defining '{}' you must set '{}' as well".format(
config_key,
required_keys)
elif error.cause:
error_msg = str(error.cause)
msg_format = "{path} is invalid: {msg}"
elif error.path:
msg_format = "{path} value {msg}"
if msg_format:
return msg_format.format(path=path_string(path), msg=error_msg)
return error.message
def parse_key_from_error_msg(error):
try:
return error.message.split("'")[1]
except IndexError:
return error.message.split('(')[1].split(' ')[0].strip("'")
def path_string(path):
return ".".join(c for c in path if isinstance(c, str))
def _parse_valid_types_from_validator(validator):
"""A validator value can be either an array of valid types or a string of
a valid type. Parse the valid types and prefix with the correct article.
"""
if not isinstance(validator, list):
return anglicize_json_type(validator)
if len(validator) == 1:
return anglicize_json_type(validator[0])
return "{}, or {}".format(
", ".join([anglicize_json_type(validator[0])] + validator[1:-1]),
anglicize_json_type(validator[-1]))
def _parse_oneof_validator(error):
"""oneOf has multiple schemas, so we need to reason about which schema, sub
schema or constraint the validation is failing on.
Inspecting the context value of a ValidationError gives us information about
which sub schema failed and which kind of error it is.
"""
types = []
for context in error.context:
if context.validator == 'oneOf':
_, error_msg = _parse_oneof_validator(context)
return path_string(context.path), error_msg
if context.validator == 'required':
return (None, context.message)
if context.validator == 'additionalProperties':
invalid_config_key = parse_key_from_error_msg(context)
return (None, "contains unsupported option: '{}'".format(invalid_config_key))
if context.validator == 'uniqueItems':
return (
path_string(context.path) if context.path else None,
"contains non-unique items, please remove duplicates from {}".format(
context.instance),
)
if context.path:
return (
path_string(context.path),
"contains {}, which is an invalid type, it should be {}".format(
json.dumps(context.instance),
_parse_valid_types_from_validator(context.validator_value)),
)
if context.validator == 'type':
types.append(context.validator_value)
valid_types = _parse_valid_types_from_validator(types)
return (None, "contains an invalid type, it should be {}".format(valid_types))
def process_service_constraint_errors(error, service_name, version):
if version == V1:
if 'image' in error.instance and 'build' in error.instance:
return (
"Service {} has both an image and build path specified. "
"A service can either be built to image or use an existing "
"image, not both.".format(service_name))
if 'image' in error.instance and 'dockerfile' in error.instance:
return (
"Service {} has both an image and alternate Dockerfile. "
"A service can either be built to image or use an existing "
"image, not both.".format(service_name))
if 'image' not in error.instance and 'build' not in error.instance:
return (
"Service {} has neither an image nor a build context specified. "
"At least one must be provided.".format(service_name))
def process_config_schema_errors(error):
path = list(error.path)
if 'id' in error.schema:
error_msg = handle_error_for_schema_with_id(error, path)
if error_msg:
return error_msg
return handle_generic_error(error, path)
def keys_to_str(config_file):
"""
Non-string keys may break validator with patterned fields.
"""
d = {}
for k, v in config_file.items():
d[str(k)] = v
if isinstance(v, dict):
d[str(k)] = keys_to_str(v)
return d
def validate_against_config_schema(config_file, version):
schema = load_jsonschema(version)
config = keys_to_str(config_file.config)
format_checker = FormatChecker(["ports", "expose", "subnet_ip_address"])
validator = Draft4Validator(
schema,
resolver=RefResolver(get_resolver_path(), schema),
format_checker=format_checker)
handle_errors(
validator.iter_errors(config),
process_config_schema_errors,
config_file.filename)
def validate_service_constraints(config, service_name, config_file):
def handler(errors):
return process_service_constraint_errors(
errors, service_name, config_file.version)
schema = load_jsonschema(config_file.version)
validator = Draft4Validator(schema['definitions']['constraints']['service'])
handle_errors(validator.iter_errors(config), handler, None)
def validate_cpu(service_config):
cpus = service_config.config.get('cpus')
if not cpus:
return
nano_cpus = cpus * NANOCPUS_SCALE
if isinstance(nano_cpus, float) and not nano_cpus.is_integer():
raise ConfigurationError(
"cpus must have nine or less digits after decimal point")
def get_schema_path():
return os.path.dirname(os.path.abspath(__file__))
def load_jsonschema(version):
name = "compose_spec"
if version == V1:
name = "config_schema_v1"
filename = os.path.join(
get_schema_path(),
"{}.json".format(name))
if not os.path.exists(filename):
raise ConfigurationError(
'Version in "{}" is unsupported. {}'
.format(filename, VERSION_EXPLANATION))
with open(filename) as fh:
return json.load(fh)
def get_resolver_path():
schema_path = get_schema_path()
if sys.platform == "win32":
scheme = "///"
# TODO: why is this necessary?
schema_path = schema_path.replace('\\', '/')
else:
scheme = "//"
return "file:{}{}/".format(scheme, schema_path)
def handle_errors(errors, format_error_func, filename):
"""jsonschema returns an error tree full of information to explain what has
gone wrong. Process each error and pull out relevant information and re-write
helpful error messages that are relevant.
"""
errors = sorted(errors, key=str)
if not errors:
return
error_msg = '\n'.join(format_error_func(error) for error in errors)
raise ConfigurationError(
"The Compose file{file_msg} is invalid because:\n{error_msg}".format(
file_msg=" '{}'".format(filename) if filename else "",
error_msg=error_msg))
def validate_healthcheck(service_config):
healthcheck = service_config.config.get('healthcheck', {})
if 'test' in healthcheck and isinstance(healthcheck['test'], list):
if len(healthcheck['test']) == 0:
raise ConfigurationError(
'Service "{}" defines an invalid healthcheck: '
'"test" is an empty list'
.format(service_config.name))
# when disable is true config.py::process_healthcheck adds "test: ['NONE']" to service_config
elif healthcheck['test'][0] == 'NONE' and len(healthcheck) > 1:
raise ConfigurationError(
'Service "{}" defines an invalid healthcheck: '
'"disable: true" cannot be combined with other options'
.format(service_config.name))
elif healthcheck['test'][0] not in ('NONE', 'CMD', 'CMD-SHELL'):
raise ConfigurationError(
'Service "{}" defines an invalid healthcheck: '
'when "test" is a list the first item must be either NONE, CMD or CMD-SHELL'
.format(service_config.name))