diff --git a/magma/squid_cnf/charms/squid/actions/addurl b/magma/squid_cnf/charms/squid/actions/addurl deleted file mode 100755 index e12bf9fef6d3bbf2fedb07eac0f2737a14e0ae8d..0000000000000000000000000000000000000000 --- a/magma/squid_cnf/charms/squid/actions/addurl +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash - -URL=`action-get url` - -if ! grep -Fxq "http_access allow allowedurls" /etc/squid/squid.conf -then - sed -i '/^# And finally deny all .*/i http_access allow allowedurls\n' /etc/squid/squid.conf -fi - -sed -i "/^http_access allow allowedurls.*/i acl allowedurls dstdomain \.$URL" /etc/squid/squid.conf - -kill -HUP `cat /var/run/squid.pid` - diff --git a/magma/squid_cnf/charms/squid/config.yaml b/magma/squid_cnf/charms/squid/config.yaml index 65c33a86fdcdd7228c88268a6ae9a65f71efb8d3..450525eeed2cb05b0cf347357d67b3693f8c192f 100644 --- a/magma/squid_cnf/charms/squid/config.yaml +++ b/magma/squid_cnf/charms/squid/config.yaml @@ -1,9 +1,5 @@ options: - image: - type: string - description: 'Docker image for squid' - default: 'domfleischmann/squid-python' port: type: int - description: 'Port' + description: "Port" default: 3128 diff --git a/magma/squid_cnf/charms/squid/dispatch b/magma/squid_cnf/charms/squid/dispatch new file mode 100755 index 0000000000000000000000000000000000000000..fe31c0567bdce62a6542a6470997cb6a874e4bd8 --- /dev/null +++ b/magma/squid_cnf/charms/squid/dispatch @@ -0,0 +1,3 @@ +#!/bin/sh + +JUJU_DISPATCH_PATH="${JUJU_DISPATCH_PATH:-$0}" PYTHONPATH=lib:venv ./src/charm.py diff --git a/magma/squid_cnf/charms/squid/hooks/install b/magma/squid_cnf/charms/squid/hooks/install index 25b1f68fa39d58d33c08ca420c3d439d19be0c55..8b970447af1decd19c27ca3c609fc97f56a233e3 120000 --- a/magma/squid_cnf/charms/squid/hooks/install +++ b/magma/squid_cnf/charms/squid/hooks/install @@ -1 +1 @@ -../src/charm.py \ No newline at end of file +../dispatch \ No newline at end of file diff --git a/magma/squid_cnf/charms/squid/hooks/start b/magma/squid_cnf/charms/squid/hooks/start new file mode 120000 index 0000000000000000000000000000000000000000..8b970447af1decd19c27ca3c609fc97f56a233e3 --- /dev/null +++ b/magma/squid_cnf/charms/squid/hooks/start @@ -0,0 +1 @@ +../dispatch \ No newline at end of file diff --git a/magma/squid_cnf/charms/squid/hooks/upgrade-charm b/magma/squid_cnf/charms/squid/hooks/upgrade-charm new file mode 120000 index 0000000000000000000000000000000000000000..8b970447af1decd19c27ca3c609fc97f56a233e3 --- /dev/null +++ b/magma/squid_cnf/charms/squid/hooks/upgrade-charm @@ -0,0 +1 @@ +../dispatch \ No newline at end of file diff --git a/magma/squid_cnf/charms/squid/lib/.empty b/magma/squid_cnf/charms/squid/lib/.empty deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/magma/squid_cnf/charms/squid/lib/ops/__init__.py b/magma/squid_cnf/charms/squid/lib/ops/__init__.py deleted file mode 100644 index b2089e3803186b5a0b12e9423b1c452596adf3c6..0000000000000000000000000000000000000000 --- a/magma/squid_cnf/charms/squid/lib/ops/__init__.py +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright 2020 Canonical Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""The Operator Framework.""" - -# Import here the bare minimum to break the circular import between modules -from . import charm # NOQA diff --git a/magma/squid_cnf/charms/squid/lib/ops/charm.py b/magma/squid_cnf/charms/squid/lib/ops/charm.py deleted file mode 100755 index 4682c20e9ff9c41db6ef748a2fd38fecdd331148..0000000000000000000000000000000000000000 --- a/magma/squid_cnf/charms/squid/lib/ops/charm.py +++ /dev/null @@ -1,562 +0,0 @@ -# Copyright 2019-2020 Canonical Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -import pathlib -import typing - -import yaml - -from ops.framework import Object, EventSource, EventBase, Framework, ObjectEvents -from ops import model - - -class HookEvent(EventBase): - """A base class for events that trigger because of a Juju hook firing.""" - - -class ActionEvent(EventBase): - """A base class for events that trigger when a user asks for an Action to be run. - - To read the parameters for the action, see the instance variable `params`. - To respond with the result of the action, call `set_results`. To add progress - messages that are visible as the action is progressing use `log`. - - :ivar params: The parameters passed to the action (read by action-get) - """ - - def defer(self): - """Action events are not deferable like other events. - - This is because an action runs synchronously and the user is waiting for the result. - """ - raise RuntimeError('cannot defer action events') - - def restore(self, snapshot: dict) -> None: - """Used by the operator framework to record the action. - - Not meant to be called directly by Charm code. - """ - env_action_name = os.environ.get('JUJU_ACTION_NAME') - event_action_name = self.handle.kind[:-len('_action')].replace('_', '-') - if event_action_name != env_action_name: - # This could only happen if the dev manually emits the action, or from a bug. - raise RuntimeError('action event kind does not match current action') - # Params are loaded at restore rather than __init__ because - # the model is not available in __init__. - self.params = self.framework.model._backend.action_get() - - def set_results(self, results: typing.Mapping) -> None: - """Report the result of the action. - - Args: - results: The result of the action as a Dict - """ - self.framework.model._backend.action_set(results) - - def log(self, message: str) -> None: - """Send a message that a user will see while the action is running. - - Args: - message: The message for the user. - """ - self.framework.model._backend.action_log(message) - - def fail(self, message: str = '') -> None: - """Report that this action has failed. - - Args: - message: Optional message to record why it has failed. - """ - self.framework.model._backend.action_fail(message) - - -class InstallEvent(HookEvent): - """Represents the `install` hook from Juju.""" - - -class StartEvent(HookEvent): - """Represents the `start` hook from Juju.""" - - -class StopEvent(HookEvent): - """Represents the `stop` hook from Juju.""" - - -class RemoveEvent(HookEvent): - """Represents the `remove` hook from Juju. """ - - -class ConfigChangedEvent(HookEvent): - """Represents the `config-changed` hook from Juju.""" - - -class UpdateStatusEvent(HookEvent): - """Represents the `update-status` hook from Juju.""" - - -class UpgradeCharmEvent(HookEvent): - """Represents the `upgrade-charm` hook from Juju. - - This will be triggered when a user has run `juju upgrade-charm`. It is run after Juju - has unpacked the upgraded charm code, and so this event will be handled with new code. - """ - - -class PreSeriesUpgradeEvent(HookEvent): - """Represents the `pre-series-upgrade` hook from Juju. - - This happens when a user has run `juju upgrade-series MACHINE prepare` and - will fire for each unit that is running on the machine, telling them that - the user is preparing to upgrade the Machine's series (eg trusty->bionic). - The charm should take actions to prepare for the upgrade (a database charm - would want to write out a version-independent dump of the database, so that - when a new version of the database is available in a new series, it can be - used.) - Once all units on a machine have run `pre-series-upgrade`, the user will - initiate the steps to actually upgrade the machine (eg `do-release-upgrade`). - When the upgrade has been completed, the :class:`PostSeriesUpgradeEvent` will fire. - """ - - -class PostSeriesUpgradeEvent(HookEvent): - """Represents the `post-series-upgrade` hook from Juju. - - This is run after the user has done a distribution upgrade (or rolled back - and kept the same series). It is called in response to - `juju upgrade-series MACHINE complete`. Charms are expected to do whatever - steps are necessary to reconfigure their applications for the new series. - """ - - -class LeaderElectedEvent(HookEvent): - """Represents the `leader-elected` hook from Juju. - - Juju will trigger this when a new lead unit is chosen for a given application. - This represents the leader of the charm information (not necessarily the primary - of a running application). The main utility is that charm authors can know - that only one unit will be a leader at any given time, so they can do - configuration, etc, that would otherwise require coordination between units. - (eg, selecting a password for a new relation) - """ - - -class LeaderSettingsChangedEvent(HookEvent): - """Represents the `leader-settings-changed` hook from Juju. - - Deprecated. This represents when a lead unit would call `leader-set` to inform - the other units of an application that they have new information to handle. - This has been deprecated in favor of using a Peer relation, and having the - leader set a value in the Application data bag for that peer relation. - (see :class:`RelationChangedEvent`). - """ - - -class CollectMetricsEvent(HookEvent): - """Represents the `collect-metrics` hook from Juju. - - Note that events firing during a CollectMetricsEvent are currently - sandboxed in how they can interact with Juju. To report metrics - use :meth:`.add_metrics`. - """ - - def add_metrics(self, metrics: typing.Mapping, labels: typing.Mapping = None) -> None: - """Record metrics that have been gathered by the charm for this unit. - - Args: - metrics: A collection of {key: float} pairs that contains the - metrics that have been gathered - labels: {key:value} strings that can be applied to the - metrics that are being gathered - """ - self.framework.model._backend.add_metrics(metrics, labels) - - -class RelationEvent(HookEvent): - """A base class representing the various relation lifecycle events. - - Charmers should not be creating RelationEvents directly. The events will be - generated by the framework from Juju related events. Users can observe them - from the various `CharmBase.on[relation_name].relation_*` events. - - Attributes: - relation: The Relation involved in this event - app: The remote application that has triggered this event - unit: The remote unit that has triggered this event. This may be None - if the relation event was triggered as an Application level event - """ - - def __init__(self, handle, relation, app=None, unit=None): - super().__init__(handle) - - if unit is not None and unit.app != app: - raise RuntimeError( - 'cannot create RelationEvent with application {} and unit {}'.format(app, unit)) - - self.relation = relation - self.app = app - self.unit = unit - - def snapshot(self) -> dict: - """Used by the framework to serialize the event to disk. - - Not meant to be called by Charm code. - """ - snapshot = { - 'relation_name': self.relation.name, - 'relation_id': self.relation.id, - } - if self.app: - snapshot['app_name'] = self.app.name - if self.unit: - snapshot['unit_name'] = self.unit.name - return snapshot - - def restore(self, snapshot: dict) -> None: - """Used by the framework to deserialize the event from disk. - - Not meant to be called by Charm code. - """ - self.relation = self.framework.model.get_relation( - snapshot['relation_name'], snapshot['relation_id']) - - app_name = snapshot.get('app_name') - if app_name: - self.app = self.framework.model.get_app(app_name) - else: - self.app = None - - unit_name = snapshot.get('unit_name') - if unit_name: - self.unit = self.framework.model.get_unit(unit_name) - else: - self.unit = None - - -class RelationCreatedEvent(RelationEvent): - """Represents the `relation-created` hook from Juju. - - This is triggered when a new relation to another app is added in Juju. This - can occur before units for those applications have started. All existing - relations should be established before start. - """ - - -class RelationJoinedEvent(RelationEvent): - """Represents the `relation-joined` hook from Juju. - - This is triggered whenever a new unit of a related application joins the relation. - (eg, a unit was added to an existing related app, or a new relation was established - with an application that already had units.) - """ - - -class RelationChangedEvent(RelationEvent): - """Represents the `relation-changed` hook from Juju. - - This is triggered whenever there is a change to the data bucket for a related - application or unit. Look at `event.relation.data[event.unit/app]` to see the - new information. - """ - - -class RelationDepartedEvent(RelationEvent): - """Represents the `relation-departed` hook from Juju. - - This is the inverse of the RelationJoinedEvent, representing when a unit - is leaving the relation (the unit is being removed, the app is being removed, - the relation is being removed). It is fired once for each unit that is - going away. - """ - - -class RelationBrokenEvent(RelationEvent): - """Represents the `relation-broken` hook from Juju. - - If a relation is being removed (`juju remove-relation` or `juju remove-application`), - once all the units have been removed, RelationBrokenEvent will fire to signal - that the relationship has been fully terminated. - """ - - -class StorageEvent(HookEvent): - """Base class representing Storage related events.""" - - -class StorageAttachedEvent(StorageEvent): - """Represents the `storage-attached` hook from Juju. - - Called when new storage is available for the charm to use. - """ - - -class StorageDetachingEvent(StorageEvent): - """Represents the `storage-detaching` hook from Juju. - - Called when storage a charm has been using is going away. - """ - - -class CharmEvents(ObjectEvents): - """The events that are generated by Juju in response to the lifecycle of an application.""" - - install = EventSource(InstallEvent) - start = EventSource(StartEvent) - stop = EventSource(StopEvent) - remove = EventSource(RemoveEvent) - update_status = EventSource(UpdateStatusEvent) - config_changed = EventSource(ConfigChangedEvent) - upgrade_charm = EventSource(UpgradeCharmEvent) - pre_series_upgrade = EventSource(PreSeriesUpgradeEvent) - post_series_upgrade = EventSource(PostSeriesUpgradeEvent) - leader_elected = EventSource(LeaderElectedEvent) - leader_settings_changed = EventSource(LeaderSettingsChangedEvent) - collect_metrics = EventSource(CollectMetricsEvent) - - -class CharmBase(Object): - """Base class that represents the Charm overall. - - Usually this initialization is done by ops.main.main() rather than Charm authors - directly instantiating a Charm. - - Args: - framework: The framework responsible for managing the Model and events for this - Charm. - key: Arbitrary key to distinguish this instance of CharmBase from another. - Generally is None when initialized by the framework. For charms instantiated by - main.main(), this is currenly None. - Attributes: - on: Defines all events that the Charm will fire. - """ - - on = CharmEvents() - - def __init__(self, framework: Framework, key: typing.Optional[str]): - """Initialize the Charm with its framework and application name. - - """ - super().__init__(framework, key) - - for relation_name in self.framework.meta.relations: - relation_name = relation_name.replace('-', '_') - self.on.define_event(relation_name + '_relation_created', RelationCreatedEvent) - self.on.define_event(relation_name + '_relation_joined', RelationJoinedEvent) - self.on.define_event(relation_name + '_relation_changed', RelationChangedEvent) - self.on.define_event(relation_name + '_relation_departed', RelationDepartedEvent) - self.on.define_event(relation_name + '_relation_broken', RelationBrokenEvent) - - for storage_name in self.framework.meta.storages: - storage_name = storage_name.replace('-', '_') - self.on.define_event(storage_name + '_storage_attached', StorageAttachedEvent) - self.on.define_event(storage_name + '_storage_detaching', StorageDetachingEvent) - - for action_name in self.framework.meta.actions: - action_name = action_name.replace('-', '_') - self.on.define_event(action_name + '_action', ActionEvent) - - @property - def app(self) -> model.Application: - """Application that this unit is part of.""" - return self.framework.model.app - - @property - def unit(self) -> model.Unit: - """Unit that this execution is responsible for.""" - return self.framework.model.unit - - @property - def meta(self) -> 'CharmMeta': - """CharmMeta of this charm. - """ - return self.framework.meta - - @property - def charm_dir(self) -> pathlib.Path: - """Root directory of the Charm as it is running. - """ - return self.framework.charm_dir - - -class CharmMeta: - """Object containing the metadata for the charm. - - This is read from metadata.yaml and/or actions.yaml. Generally charms will - define this information, rather than reading it at runtime. This class is - mostly for the framework to understand what the charm has defined. - - The maintainers, tags, terms, series, and extra_bindings attributes are all - lists of strings. The requires, provides, peers, relations, storage, - resources, and payloads attributes are all mappings of names to instances - of the respective RelationMeta, StorageMeta, ResourceMeta, or PayloadMeta. - - The relations attribute is a convenience accessor which includes all of the - requires, provides, and peers RelationMeta items. If needed, the role of - the relation definition can be obtained from its role attribute. - - Attributes: - name: The name of this charm - summary: Short description of what this charm does - description: Long description for this charm - maintainers: A list of strings of the email addresses of the maintainers - of this charm. - tags: Charm store tag metadata for categories associated with this charm. - terms: Charm store terms that should be agreed to before this charm can - be deployed. (Used for things like licensing issues.) - series: The list of supported OS series that this charm can support. - The first entry in the list is the default series that will be - used by deploy if no other series is requested by the user. - subordinate: True/False whether this charm is intended to be used as a - subordinate charm. - min_juju_version: If supplied, indicates this charm needs features that - are not available in older versions of Juju. - requires: A dict of {name: :class:`RelationMeta` } for each 'requires' relation. - provides: A dict of {name: :class:`RelationMeta` } for each 'provides' relation. - peers: A dict of {name: :class:`RelationMeta` } for each 'peer' relation. - relations: A dict containing all :class:`RelationMeta` attributes (merged from other - sections) - storages: A dict of {name: :class:`StorageMeta`} for each defined storage. - resources: A dict of {name: :class:`ResourceMeta`} for each defined resource. - payloads: A dict of {name: :class:`PayloadMeta`} for each defined payload. - extra_bindings: A dict of additional named bindings that a charm can use - for network configuration. - actions: A dict of {name: :class:`ActionMeta`} for actions that the charm has defined. - Args: - raw: a mapping containing the contents of metadata.yaml - actions_raw: a mapping containing the contents of actions.yaml - """ - - def __init__(self, raw: dict = {}, actions_raw: dict = {}): - self.name = raw.get('name', '') - self.summary = raw.get('summary', '') - self.description = raw.get('description', '') - self.maintainers = [] - if 'maintainer' in raw: - self.maintainers.append(raw['maintainer']) - if 'maintainers' in raw: - self.maintainers.extend(raw['maintainers']) - self.tags = raw.get('tags', []) - self.terms = raw.get('terms', []) - self.series = raw.get('series', []) - self.subordinate = raw.get('subordinate', False) - self.min_juju_version = raw.get('min-juju-version') - self.requires = {name: RelationMeta('requires', name, rel) - for name, rel in raw.get('requires', {}).items()} - self.provides = {name: RelationMeta('provides', name, rel) - for name, rel in raw.get('provides', {}).items()} - # TODO: (jam 2020-05-11) The *role* should be 'peer' even though it comes from the - # 'peers' section. - self.peers = {name: RelationMeta('peers', name, rel) - for name, rel in raw.get('peers', {}).items()} - self.relations = {} - self.relations.update(self.requires) - self.relations.update(self.provides) - self.relations.update(self.peers) - self.storages = {name: StorageMeta(name, storage) - for name, storage in raw.get('storage', {}).items()} - self.resources = {name: ResourceMeta(name, res) - for name, res in raw.get('resources', {}).items()} - self.payloads = {name: PayloadMeta(name, payload) - for name, payload in raw.get('payloads', {}).items()} - self.extra_bindings = raw.get('extra-bindings', {}) - self.actions = {name: ActionMeta(name, action) for name, action in actions_raw.items()} - - @classmethod - def from_yaml( - cls, metadata: typing.Union[str, typing.TextIO], - actions: typing.Optional[typing.Union[str, typing.TextIO]] = None): - """Instantiate a CharmMeta from a YAML description of metadata.yaml. - - Args: - metadata: A YAML description of charm metadata (name, relations, etc.) - This can be a simple string, or a file-like object. (passed to `yaml.safe_load`). - actions: YAML description of Actions for this charm (eg actions.yaml) - """ - meta = yaml.safe_load(metadata) - raw_actions = {} - if actions is not None: - raw_actions = yaml.safe_load(actions) - return cls(meta, raw_actions) - - -class RelationMeta: - """Object containing metadata about a relation definition. - - Should not be constructed directly by Charm code. Is gotten from one of - :attr:`CharmMeta.peers`, :attr:`CharmMeta.requires`, :attr:`CharmMeta.provides`, - :attr:`CharmMeta.relations`. - - Attributes: - role: This is one of requires/provides/peers - relation_name: Name of this relation from metadata.yaml - interface_name: Optional definition of the interface protocol. - scope: "global" or "container" scope based on how the relation should be used. - """ - - def __init__(self, role, relation_name, raw): - self.role = role - self.relation_name = relation_name - self.interface_name = raw['interface'] - self.scope = raw.get('scope') - - -class StorageMeta: - """Object containing metadata about a storage definition.""" - - def __init__(self, name, raw): - self.storage_name = name - self.type = raw['type'] - self.description = raw.get('description', '') - self.shared = raw.get('shared', False) - self.read_only = raw.get('read-only', False) - self.minimum_size = raw.get('minimum-size') - self.location = raw.get('location') - self.multiple_range = None - if 'multiple' in raw: - range = raw['multiple']['range'] - if '-' not in range: - self.multiple_range = (int(range), int(range)) - else: - range = range.split('-') - self.multiple_range = (int(range[0]), int(range[1]) if range[1] else None) - - -class ResourceMeta: - """Object containing metadata about a resource definition.""" - - def __init__(self, name, raw): - self.resource_name = name - self.type = raw['type'] - self.filename = raw.get('filename', None) - self.description = raw.get('description', '') - - -class PayloadMeta: - """Object containing metadata about a payload definition.""" - - def __init__(self, name, raw): - self.payload_name = name - self.type = raw['type'] - - -class ActionMeta: - """Object containing metadata about an action's definition.""" - - def __init__(self, name, raw=None): - raw = raw or {} - self.name = name - self.title = raw.get('title', '') - self.description = raw.get('description', '') - self.parameters = raw.get('params', {}) # {: } - self.required = raw.get('required', []) # [, ...] diff --git a/magma/squid_cnf/charms/squid/lib/ops/main.py b/magma/squid_cnf/charms/squid/lib/ops/main.py deleted file mode 100755 index 0f5391d76e45ba32dc652adfc99b2c7716d8af36..0000000000000000000000000000000000000000 --- a/magma/squid_cnf/charms/squid/lib/ops/main.py +++ /dev/null @@ -1,235 +0,0 @@ -#!/usr/bin/env python3 -# Copyright 2019 Canonical Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -import subprocess -import sys -from pathlib import Path - -import yaml - -import ops.charm -import ops.framework -import ops.model -import logging - -from ops.log import setup_root_logging - -CHARM_STATE_FILE = '.unit-state.db' - - -logger = logging.getLogger() - - -def _get_charm_dir(): - charm_dir = os.environ.get("JUJU_CHARM_DIR") - if charm_dir is None: - # Assume $JUJU_CHARM_DIR/lib/op/main.py structure. - charm_dir = Path('{}/../../..'.format(__file__)).resolve() - else: - charm_dir = Path(charm_dir).resolve() - return charm_dir - - -def _load_metadata(charm_dir): - metadata = yaml.safe_load((charm_dir / 'metadata.yaml').read_text()) - - actions_meta = charm_dir / 'actions.yaml' - if actions_meta.exists(): - actions_metadata = yaml.safe_load(actions_meta.read_text()) - else: - actions_metadata = {} - return metadata, actions_metadata - - -def _create_event_link(charm, bound_event): - """Create a symlink for a particular event. - - charm -- A charm object. - bound_event -- An event for which to create a symlink. - """ - if issubclass(bound_event.event_type, ops.charm.HookEvent): - event_dir = charm.framework.charm_dir / 'hooks' - event_path = event_dir / bound_event.event_kind.replace('_', '-') - elif issubclass(bound_event.event_type, ops.charm.ActionEvent): - if not bound_event.event_kind.endswith("_action"): - raise RuntimeError( - 'action event name {} needs _action suffix'.format(bound_event.event_kind)) - event_dir = charm.framework.charm_dir / 'actions' - # The event_kind is suffixed with "_action" while the executable is not. - event_path = event_dir / bound_event.event_kind[:-len('_action')].replace('_', '-') - else: - raise RuntimeError( - 'cannot create a symlink: unsupported event type {}'.format(bound_event.event_type)) - - event_dir.mkdir(exist_ok=True) - if not event_path.exists(): - # CPython has different implementations for populating sys.argv[0] for Linux and Windows. - # For Windows it is always an absolute path (any symlinks are resolved) - # while for Linux it can be a relative path. - target_path = os.path.relpath(os.path.realpath(sys.argv[0]), str(event_dir)) - - # Ignore the non-symlink files or directories - # assuming the charm author knows what they are doing. - logger.debug( - 'Creating a new relative symlink at %s pointing to %s', - event_path, target_path) - event_path.symlink_to(target_path) - - -def _setup_event_links(charm_dir, charm): - """Set up links for supported events that originate from Juju. - - Whether a charm can handle an event or not can be determined by - introspecting which events are defined on it. - - Hooks or actions are created as symlinks to the charm code file - which is determined by inspecting symlinks provided by the charm - author at hooks/install or hooks/start. - - charm_dir -- A root directory of the charm. - charm -- An instance of the Charm class. - - """ - for bound_event in charm.on.events().values(): - # Only events that originate from Juju need symlinks. - if issubclass(bound_event.event_type, (ops.charm.HookEvent, ops.charm.ActionEvent)): - _create_event_link(charm, bound_event) - - -def _emit_charm_event(charm, event_name): - """Emits a charm event based on a Juju event name. - - charm -- A charm instance to emit an event from. - event_name -- A Juju event name to emit on a charm. - """ - event_to_emit = None - try: - event_to_emit = getattr(charm.on, event_name) - except AttributeError: - logger.debug("event %s not defined for %s", event_name, charm) - - # If the event is not supported by the charm implementation, do - # not error out or try to emit it. This is to support rollbacks. - if event_to_emit is not None: - args, kwargs = _get_event_args(charm, event_to_emit) - logger.debug('Emitting Juju event %s', event_name) - event_to_emit.emit(*args, **kwargs) - - -def _get_event_args(charm, bound_event): - event_type = bound_event.event_type - model = charm.framework.model - - if issubclass(event_type, ops.charm.RelationEvent): - relation_name = os.environ['JUJU_RELATION'] - relation_id = int(os.environ['JUJU_RELATION_ID'].split(':')[-1]) - relation = model.get_relation(relation_name, relation_id) - else: - relation = None - - remote_app_name = os.environ.get('JUJU_REMOTE_APP', '') - remote_unit_name = os.environ.get('JUJU_REMOTE_UNIT', '') - if remote_app_name or remote_unit_name: - if not remote_app_name: - if '/' not in remote_unit_name: - raise RuntimeError('invalid remote unit name: {}'.format(remote_unit_name)) - remote_app_name = remote_unit_name.split('/')[0] - args = [relation, model.get_app(remote_app_name)] - if remote_unit_name: - args.append(model.get_unit(remote_unit_name)) - return args, {} - elif relation: - return [relation], {} - return [], {} - - -def main(charm_class): - """Setup the charm and dispatch the observed event. - - The event name is based on the way this executable was called (argv[0]). - """ - charm_dir = _get_charm_dir() - - model_backend = ops.model.ModelBackend() - debug = ('JUJU_DEBUG' in os.environ) - setup_root_logging(model_backend, debug=debug) - - # Process the Juju event relevant to the current hook execution - # JUJU_HOOK_NAME, JUJU_FUNCTION_NAME, and JUJU_ACTION_NAME are not used - # in order to support simulation of events from debugging sessions. - # - # TODO: For Windows, when symlinks are used, this is not a valid - # method of getting an event name (see LP: #1854505). - juju_exec_path = Path(sys.argv[0]) - has_dispatch = juju_exec_path.name == 'dispatch' - if has_dispatch: - # The executable was 'dispatch', which means the actual hook we want to - # run needs to be looked up in the JUJU_DISPATCH_PATH env var, where it - # should be a path relative to the charm directory (the directory that - # holds `dispatch`). If that path actually exists, we want to run that - # before continuing. - dispatch_path = juju_exec_path.parent / Path(os.environ['JUJU_DISPATCH_PATH']) - if dispatch_path.exists() and dispatch_path.resolve() != juju_exec_path.resolve(): - argv = sys.argv.copy() - argv[0] = str(dispatch_path) - try: - subprocess.run(argv, check=True) - except subprocess.CalledProcessError as e: - logger.warning("hook %s exited with status %d", dispatch_path, e.returncode) - sys.exit(e.returncode) - juju_exec_path = dispatch_path - juju_event_name = juju_exec_path.name.replace('-', '_') - if juju_exec_path.parent.name == 'actions': - juju_event_name = '{}_action'.format(juju_event_name) - - metadata, actions_metadata = _load_metadata(charm_dir) - meta = ops.charm.CharmMeta(metadata, actions_metadata) - unit_name = os.environ['JUJU_UNIT_NAME'] - model = ops.model.Model(unit_name, meta, model_backend) - - # TODO: If Juju unit agent crashes after exit(0) from the charm code - # the framework will commit the snapshot but Juju will not commit its - # operation. - charm_state_path = charm_dir / CHARM_STATE_FILE - framework = ops.framework.Framework(charm_state_path, charm_dir, meta, model) - try: - charm = charm_class(framework, None) - - if not has_dispatch: - # When a charm is force-upgraded and a unit is in an error state Juju - # does not run upgrade-charm and instead runs the failed hook followed - # by config-changed. Given the nature of force-upgrading the hook setup - # code is not triggered on config-changed. - # - # 'start' event is included as Juju does not fire the install event for - # K8s charms (see LP: #1854635). - if (juju_event_name in ('install', 'start', 'upgrade_charm') - or juju_event_name.endswith('_storage_attached')): - _setup_event_links(charm_dir, charm) - - # TODO: Remove the collect_metrics check below as soon as the relevant - # Juju changes are made. - # - # Skip reemission of deferred events for collect-metrics events because - # they do not have the full access to all hook tools. - if juju_event_name != 'collect_metrics': - framework.reemit() - - _emit_charm_event(charm, juju_event_name) - - framework.commit() - finally: - framework.close() diff --git a/magma/squid_cnf/charms/squid/metadata.yaml b/magma/squid_cnf/charms/squid/metadata.yaml index aa3bbfa1a1fc888e1e5f0d88b0a05ae8a9ee5900..aabf09bc24512226f22974867fb69bd768fdae4d 100644 --- a/magma/squid_cnf/charms/squid/metadata.yaml +++ b/magma/squid_cnf/charms/squid/metadata.yaml @@ -14,10 +14,11 @@ deployment: type: stateful service: loadbalancer storage: - docker: - type: filesystem - location: /srv/docker/squid spool: type: filesystem location: /var/spool/squid - +resources: + image: + type: oci-image + description: OSM docker image for LCM + upstream-source: "domfleischmann/squid-python" \ No newline at end of file diff --git a/magma/squid_cnf/charms/squid/mod/.empty b/magma/squid_cnf/charms/squid/mod/.empty deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/magma/squid_cnf/charms/squid/mod/operator/.flake8 b/magma/squid_cnf/charms/squid/mod/operator/.flake8 deleted file mode 100644 index 61d908155588a7968dd25c90cff349377305a789..0000000000000000000000000000000000000000 --- a/magma/squid_cnf/charms/squid/mod/operator/.flake8 +++ /dev/null @@ -1,2 +0,0 @@ -[flake8] -max-line-length = 99 diff --git a/magma/squid_cnf/charms/squid/mod/operator/.gitignore b/magma/squid_cnf/charms/squid/mod/operator/.gitignore deleted file mode 100644 index 5596c502618d6428de0199c4cec069c151c0edd3..0000000000000000000000000000000000000000 --- a/magma/squid_cnf/charms/squid/mod/operator/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -__pycache__ -/sandbox -.idea -docs/_build diff --git a/magma/squid_cnf/charms/squid/mod/operator/.readthedocs.yaml b/magma/squid_cnf/charms/squid/mod/operator/.readthedocs.yaml deleted file mode 100644 index ee5cf263e50a8aa864554613c9aa6214ad043c0c..0000000000000000000000000000000000000000 --- a/magma/squid_cnf/charms/squid/mod/operator/.readthedocs.yaml +++ /dev/null @@ -1,7 +0,0 @@ -version: 2 # required -formats: [] # i.e. no extra formats (for now) -python: - version: "3.5" - install: - - requirements: docs/requirements.txt - - requirements: requirements.txt diff --git a/magma/squid_cnf/charms/squid/mod/operator/.travis.yml b/magma/squid_cnf/charms/squid/mod/operator/.travis.yml deleted file mode 100644 index f222ba8b2fa4603f457c6e2f2dee82c0cbbe8f11..0000000000000000000000000000000000000000 --- a/magma/squid_cnf/charms/squid/mod/operator/.travis.yml +++ /dev/null @@ -1,24 +0,0 @@ -dist: bionic - -language: python - -arch: - - amd64 - - arm64 - -python: - - "3.5" - - "3.6" - - "3.7" - - "3.8" - -matrix: - include: - - os: osx - language: generic - -install: - - pip3 install pyyaml autopep8 flake8 - -script: - - ./run_tests diff --git a/magma/squid_cnf/charms/squid/mod/operator/README.md b/magma/squid_cnf/charms/squid/mod/operator/README.md deleted file mode 100644 index 347bfc8effe4b26adff4268688ade2f28d502d49..0000000000000000000000000000000000000000 --- a/magma/squid_cnf/charms/squid/mod/operator/README.md +++ /dev/null @@ -1,137 +0,0 @@ -# The Operator Framework - -The Operator Framework provides a simple, lightweight, and powerful way of encapsulating operational experience in code. - -The framework will help you to: - -* model the integration of your services -* manage the lifecycle of your application -* create reusable and scalable components -* keep your code simple and readable - -## Getting Started - -The following overall structure for your charm directory is recommended: - -``` -. -├── config.yaml -├── metadata.yaml -├── mod/ -├── lib/ -│ └── ops -> ../mod/operator/ops -├── src/ -│ └── charm.py -└── hooks/ - ├── install -> ../src/charm.py - └── start -> ../src/charm.py # for k8s charms per below -``` - -The `mod/` directory should contain the operator framework dependency as a git -submodule: - -``` -git submodule add https://github.com/canonical/operator mod/operator -``` - -Then symlink from the git submodule for the operator framework into the `lib/` -directory of your charm so it can be imported at run time: - -``` -ln -s ../mod/operator/ops lib/ops -``` - -Other dependencies included as git submodules can be added in the `mod/` -directory and symlinked into `lib/` as well. - -You can sync subsequent changes from the framework and other submodule -dependencies by running: - -``` -git submodule update -``` - -Those cloning and checking out the source for your charm for the first time -will need to run: - -``` -git submodule update --init -``` - -Your `src/charm.py` is the entry point for your charm logic. It should be set -to executable and use Python 3.6 or greater. At a minimum, it needs to define -a subclass of `CharmBase` and pass that into the framework's `main` function: - -```python -import sys -sys.path.append('lib') # noqa: E402 - -from ops.charm import CharmBase -from ops.main import main - - -class MyCharm(CharmBase): - pass - - -if __name__ == "__main__": - main(MyCharm) -``` - -This charm does nothing, because the `MyCharm` class passed to the operator -framework's `main` function is empty. Functionality can be added to the charm -by instructing it to observe particular Juju events when the `MyCharm` object -is initialized. For example, - -```python -class MyCharm(CharmBase): - def __init__(self, *args): - super().__init__(*args) - self.framework.observe(self.on.start, self.on_start) - - def on_start(self, event): - # Handle the start event here. -``` - -Every standard event in Juju may be observed that way, and you can also easily -define your own events in your custom types. - -> The second argument to `observe` can be either the handler as a bound -> method, or the observer itself if the handler is a method of the observer -> that follows the conventional naming pattern. That is, in this case, we -> could have called just `self.framework.observe(self.on.start, self)`. - -The `hooks/` directory must contain a symlink to your `src/charm.py` entry -point so that Juju can call it. You only need to set up the `hooks/install` link -(`hooks/start` for K8s charms, until [lp#1854635](https://bugs.launchpad.net/juju/+bug/1854635) -is resolved), and the framework will create all others at runtime. - -Once your charm is ready, upload it to the charm store and deploy it as -normal with: - -``` -# Replace ${CHARM} with the name of the charm. -charm push . cs:~${USER}/${CHARM} -# Replace ${VERSION} with the version created by `charm push`. -charm release cs:~${USER}/${CHARM}-${VERSION} -charm grant cs:~${USER}/${CHARM}-${VERSION} everyone -# And now deploy your charm. -juju deploy cs:~${USER}/$CHARM -``` - -Alternatively, to deploy directly from local disk, run: - -``` -juju deploy . -``` - -# Operator Framework development - -If you want to work in the framework *itself* you will need the following depenencies installed in your system: - -- Python >= 3.5 -- PyYAML -- autopep8 -- flake8 - -Then you can try `./run_tests`, it should all go green. diff --git a/magma/squid_cnf/charms/squid/mod/operator/build_docs b/magma/squid_cnf/charms/squid/mod/operator/build_docs deleted file mode 100755 index af8b892f7568bdda5ac892beaeafad5696e607d3..0000000000000000000000000000000000000000 --- a/magma/squid_cnf/charms/squid/mod/operator/build_docs +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/sh - -set -e - -flavour=html - -if [ "$1" ]; then - if [ "$1" = "--help" ] || [ "$1" = "-h" ]; then - flavour=help - else - flavour="$1" - fi - shift -fi - -cd docs - -sphinx-build -M "$flavour" . _build "$@" diff --git a/magma/squid_cnf/charms/squid/mod/operator/docs/conf.py b/magma/squid_cnf/charms/squid/mod/operator/docs/conf.py deleted file mode 100644 index bd78ed2a505f7fe66a0c4e86c0e0708beba4bff9..0000000000000000000000000000000000000000 --- a/magma/squid_cnf/charms/squid/mod/operator/docs/conf.py +++ /dev/null @@ -1,89 +0,0 @@ -# Configuration file for the Sphinx documentation builder. -# -# For a full list of options see the documentation: -# https://www.sphinx-doc.org/en/master/usage/configuration.html - - -# -- Path setup -------------------------------------------------------------- - -from pathlib import Path -import sys -sys.path.insert(0, str(Path(__file__).parent.parent)) - - -# -- Project information ----------------------------------------------------- - -project = 'The Operator Framework' -copyright = '2019-2020, Canonical Ltd.' -author = 'Canonical Ltd' - - -# -- General configuration --------------------------------------------------- - -# Add any Sphinx extension module names here, as strings. They can be -# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom -# ones. -extensions = [ - 'sphinx.ext.autodoc', - 'sphinx.ext.napoleon', - 'sphinx.ext.todo', - 'sphinx.ext.viewcode', -] - -# The document name of the “master” document, that is, the document -# that contains the root toctree directive. -master_doc = 'index' - -# Add any paths that contain templates here, relative to this directory. -templates_path = ['_templates'] - -# List of patterns, relative to source directory, that match files and -# directories to ignore when looking for source files. -# This pattern also affects html_static_path and html_extra_path. -exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] - - -# -- Options for HTML output ------------------------------------------------- - -# The theme to use for HTML and HTML Help pages. See the documentation for -# a list of builtin themes. -# -# html_theme = 'nature' # 'alabaster' - -# Add any paths that contain custom static files (such as style sheets) here, -# relative to this directory. They are copied after the builtin static files, -# so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['_static'] - - -# -- Options for sphinx.ext.todo --------------------------------------------- - -# If this is True, todo and todolist produce output, else they -# produce nothing. The default is False. -todo_include_todos = False - - -# -- Options for sphinx.ext.autodoc ------------------------------------------ - -# This value controls how to represents typehints. The setting takes the -# following values: -# 'signature' – Show typehints as its signature (default) -# 'description' – Show typehints as content of function or method -# 'none' – Do not show typehints -autodoc_typehints = 'description' - -# This value selects what content will be inserted into the main body of an -# autoclass directive. The possible values are: -# 'class' - Only the class’ docstring is inserted. This is the -# default. You can still document __init__ as a separate method -# using automethod or the members option to autoclass. -# 'both' - Both the class’ and the __init__ method’s docstring are -# concatenated and inserted. -# 'init' - Only the __init__ method’s docstring is inserted. -autoclass_content = 'both' - -autodoc_default_options = { - 'members': None, # None here means "yes" - 'undoc-members': None, - 'show-inheritance': None, -} diff --git a/magma/squid_cnf/charms/squid/mod/operator/docs/index.rst b/magma/squid_cnf/charms/squid/mod/operator/docs/index.rst deleted file mode 100644 index 424d78d423e1e7dd5d4049c9f88ffaec994c1714..0000000000000000000000000000000000000000 --- a/magma/squid_cnf/charms/squid/mod/operator/docs/index.rst +++ /dev/null @@ -1,58 +0,0 @@ - -Welcome to The Operator Framework's documentation! -================================================== - -.. toctree:: - :maxdepth: 2 - :caption: Contents: - -ops package -=========== - -.. automodule:: ops - -Submodules ----------- - -ops.charm module ----------------- - -.. automodule:: ops.charm - -ops.framework module --------------------- - -.. automodule:: ops.framework - -ops.jujuversion module ----------------------- - -.. automodule:: ops.jujuversion - -ops.log module --------------- - -.. automodule:: ops.log - -ops.main module ---------------- - -.. automodule:: ops.main - -ops.model module ----------------- - -.. automodule:: ops.model - -ops.testing module ------------------- - -.. automodule:: ops.testing - - -Indices and tables -================== - -* :ref:`genindex` -* :ref:`modindex` -* :ref:`search` diff --git a/magma/squid_cnf/charms/squid/mod/operator/docs/requirements.txt b/magma/squid_cnf/charms/squid/mod/operator/docs/requirements.txt deleted file mode 100644 index 37b11036df8f354e8510b77a87bba554cfafe887..0000000000000000000000000000000000000000 --- a/magma/squid_cnf/charms/squid/mod/operator/docs/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -sphinx<2 diff --git a/magma/squid_cnf/charms/squid/mod/operator/ops/charm.py b/magma/squid_cnf/charms/squid/mod/operator/ops/charm.py deleted file mode 100755 index 4682c20e9ff9c41db6ef748a2fd38fecdd331148..0000000000000000000000000000000000000000 --- a/magma/squid_cnf/charms/squid/mod/operator/ops/charm.py +++ /dev/null @@ -1,562 +0,0 @@ -# Copyright 2019-2020 Canonical Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -import pathlib -import typing - -import yaml - -from ops.framework import Object, EventSource, EventBase, Framework, ObjectEvents -from ops import model - - -class HookEvent(EventBase): - """A base class for events that trigger because of a Juju hook firing.""" - - -class ActionEvent(EventBase): - """A base class for events that trigger when a user asks for an Action to be run. - - To read the parameters for the action, see the instance variable `params`. - To respond with the result of the action, call `set_results`. To add progress - messages that are visible as the action is progressing use `log`. - - :ivar params: The parameters passed to the action (read by action-get) - """ - - def defer(self): - """Action events are not deferable like other events. - - This is because an action runs synchronously and the user is waiting for the result. - """ - raise RuntimeError('cannot defer action events') - - def restore(self, snapshot: dict) -> None: - """Used by the operator framework to record the action. - - Not meant to be called directly by Charm code. - """ - env_action_name = os.environ.get('JUJU_ACTION_NAME') - event_action_name = self.handle.kind[:-len('_action')].replace('_', '-') - if event_action_name != env_action_name: - # This could only happen if the dev manually emits the action, or from a bug. - raise RuntimeError('action event kind does not match current action') - # Params are loaded at restore rather than __init__ because - # the model is not available in __init__. - self.params = self.framework.model._backend.action_get() - - def set_results(self, results: typing.Mapping) -> None: - """Report the result of the action. - - Args: - results: The result of the action as a Dict - """ - self.framework.model._backend.action_set(results) - - def log(self, message: str) -> None: - """Send a message that a user will see while the action is running. - - Args: - message: The message for the user. - """ - self.framework.model._backend.action_log(message) - - def fail(self, message: str = '') -> None: - """Report that this action has failed. - - Args: - message: Optional message to record why it has failed. - """ - self.framework.model._backend.action_fail(message) - - -class InstallEvent(HookEvent): - """Represents the `install` hook from Juju.""" - - -class StartEvent(HookEvent): - """Represents the `start` hook from Juju.""" - - -class StopEvent(HookEvent): - """Represents the `stop` hook from Juju.""" - - -class RemoveEvent(HookEvent): - """Represents the `remove` hook from Juju. """ - - -class ConfigChangedEvent(HookEvent): - """Represents the `config-changed` hook from Juju.""" - - -class UpdateStatusEvent(HookEvent): - """Represents the `update-status` hook from Juju.""" - - -class UpgradeCharmEvent(HookEvent): - """Represents the `upgrade-charm` hook from Juju. - - This will be triggered when a user has run `juju upgrade-charm`. It is run after Juju - has unpacked the upgraded charm code, and so this event will be handled with new code. - """ - - -class PreSeriesUpgradeEvent(HookEvent): - """Represents the `pre-series-upgrade` hook from Juju. - - This happens when a user has run `juju upgrade-series MACHINE prepare` and - will fire for each unit that is running on the machine, telling them that - the user is preparing to upgrade the Machine's series (eg trusty->bionic). - The charm should take actions to prepare for the upgrade (a database charm - would want to write out a version-independent dump of the database, so that - when a new version of the database is available in a new series, it can be - used.) - Once all units on a machine have run `pre-series-upgrade`, the user will - initiate the steps to actually upgrade the machine (eg `do-release-upgrade`). - When the upgrade has been completed, the :class:`PostSeriesUpgradeEvent` will fire. - """ - - -class PostSeriesUpgradeEvent(HookEvent): - """Represents the `post-series-upgrade` hook from Juju. - - This is run after the user has done a distribution upgrade (or rolled back - and kept the same series). It is called in response to - `juju upgrade-series MACHINE complete`. Charms are expected to do whatever - steps are necessary to reconfigure their applications for the new series. - """ - - -class LeaderElectedEvent(HookEvent): - """Represents the `leader-elected` hook from Juju. - - Juju will trigger this when a new lead unit is chosen for a given application. - This represents the leader of the charm information (not necessarily the primary - of a running application). The main utility is that charm authors can know - that only one unit will be a leader at any given time, so they can do - configuration, etc, that would otherwise require coordination between units. - (eg, selecting a password for a new relation) - """ - - -class LeaderSettingsChangedEvent(HookEvent): - """Represents the `leader-settings-changed` hook from Juju. - - Deprecated. This represents when a lead unit would call `leader-set` to inform - the other units of an application that they have new information to handle. - This has been deprecated in favor of using a Peer relation, and having the - leader set a value in the Application data bag for that peer relation. - (see :class:`RelationChangedEvent`). - """ - - -class CollectMetricsEvent(HookEvent): - """Represents the `collect-metrics` hook from Juju. - - Note that events firing during a CollectMetricsEvent are currently - sandboxed in how they can interact with Juju. To report metrics - use :meth:`.add_metrics`. - """ - - def add_metrics(self, metrics: typing.Mapping, labels: typing.Mapping = None) -> None: - """Record metrics that have been gathered by the charm for this unit. - - Args: - metrics: A collection of {key: float} pairs that contains the - metrics that have been gathered - labels: {key:value} strings that can be applied to the - metrics that are being gathered - """ - self.framework.model._backend.add_metrics(metrics, labels) - - -class RelationEvent(HookEvent): - """A base class representing the various relation lifecycle events. - - Charmers should not be creating RelationEvents directly. The events will be - generated by the framework from Juju related events. Users can observe them - from the various `CharmBase.on[relation_name].relation_*` events. - - Attributes: - relation: The Relation involved in this event - app: The remote application that has triggered this event - unit: The remote unit that has triggered this event. This may be None - if the relation event was triggered as an Application level event - """ - - def __init__(self, handle, relation, app=None, unit=None): - super().__init__(handle) - - if unit is not None and unit.app != app: - raise RuntimeError( - 'cannot create RelationEvent with application {} and unit {}'.format(app, unit)) - - self.relation = relation - self.app = app - self.unit = unit - - def snapshot(self) -> dict: - """Used by the framework to serialize the event to disk. - - Not meant to be called by Charm code. - """ - snapshot = { - 'relation_name': self.relation.name, - 'relation_id': self.relation.id, - } - if self.app: - snapshot['app_name'] = self.app.name - if self.unit: - snapshot['unit_name'] = self.unit.name - return snapshot - - def restore(self, snapshot: dict) -> None: - """Used by the framework to deserialize the event from disk. - - Not meant to be called by Charm code. - """ - self.relation = self.framework.model.get_relation( - snapshot['relation_name'], snapshot['relation_id']) - - app_name = snapshot.get('app_name') - if app_name: - self.app = self.framework.model.get_app(app_name) - else: - self.app = None - - unit_name = snapshot.get('unit_name') - if unit_name: - self.unit = self.framework.model.get_unit(unit_name) - else: - self.unit = None - - -class RelationCreatedEvent(RelationEvent): - """Represents the `relation-created` hook from Juju. - - This is triggered when a new relation to another app is added in Juju. This - can occur before units for those applications have started. All existing - relations should be established before start. - """ - - -class RelationJoinedEvent(RelationEvent): - """Represents the `relation-joined` hook from Juju. - - This is triggered whenever a new unit of a related application joins the relation. - (eg, a unit was added to an existing related app, or a new relation was established - with an application that already had units.) - """ - - -class RelationChangedEvent(RelationEvent): - """Represents the `relation-changed` hook from Juju. - - This is triggered whenever there is a change to the data bucket for a related - application or unit. Look at `event.relation.data[event.unit/app]` to see the - new information. - """ - - -class RelationDepartedEvent(RelationEvent): - """Represents the `relation-departed` hook from Juju. - - This is the inverse of the RelationJoinedEvent, representing when a unit - is leaving the relation (the unit is being removed, the app is being removed, - the relation is being removed). It is fired once for each unit that is - going away. - """ - - -class RelationBrokenEvent(RelationEvent): - """Represents the `relation-broken` hook from Juju. - - If a relation is being removed (`juju remove-relation` or `juju remove-application`), - once all the units have been removed, RelationBrokenEvent will fire to signal - that the relationship has been fully terminated. - """ - - -class StorageEvent(HookEvent): - """Base class representing Storage related events.""" - - -class StorageAttachedEvent(StorageEvent): - """Represents the `storage-attached` hook from Juju. - - Called when new storage is available for the charm to use. - """ - - -class StorageDetachingEvent(StorageEvent): - """Represents the `storage-detaching` hook from Juju. - - Called when storage a charm has been using is going away. - """ - - -class CharmEvents(ObjectEvents): - """The events that are generated by Juju in response to the lifecycle of an application.""" - - install = EventSource(InstallEvent) - start = EventSource(StartEvent) - stop = EventSource(StopEvent) - remove = EventSource(RemoveEvent) - update_status = EventSource(UpdateStatusEvent) - config_changed = EventSource(ConfigChangedEvent) - upgrade_charm = EventSource(UpgradeCharmEvent) - pre_series_upgrade = EventSource(PreSeriesUpgradeEvent) - post_series_upgrade = EventSource(PostSeriesUpgradeEvent) - leader_elected = EventSource(LeaderElectedEvent) - leader_settings_changed = EventSource(LeaderSettingsChangedEvent) - collect_metrics = EventSource(CollectMetricsEvent) - - -class CharmBase(Object): - """Base class that represents the Charm overall. - - Usually this initialization is done by ops.main.main() rather than Charm authors - directly instantiating a Charm. - - Args: - framework: The framework responsible for managing the Model and events for this - Charm. - key: Arbitrary key to distinguish this instance of CharmBase from another. - Generally is None when initialized by the framework. For charms instantiated by - main.main(), this is currenly None. - Attributes: - on: Defines all events that the Charm will fire. - """ - - on = CharmEvents() - - def __init__(self, framework: Framework, key: typing.Optional[str]): - """Initialize the Charm with its framework and application name. - - """ - super().__init__(framework, key) - - for relation_name in self.framework.meta.relations: - relation_name = relation_name.replace('-', '_') - self.on.define_event(relation_name + '_relation_created', RelationCreatedEvent) - self.on.define_event(relation_name + '_relation_joined', RelationJoinedEvent) - self.on.define_event(relation_name + '_relation_changed', RelationChangedEvent) - self.on.define_event(relation_name + '_relation_departed', RelationDepartedEvent) - self.on.define_event(relation_name + '_relation_broken', RelationBrokenEvent) - - for storage_name in self.framework.meta.storages: - storage_name = storage_name.replace('-', '_') - self.on.define_event(storage_name + '_storage_attached', StorageAttachedEvent) - self.on.define_event(storage_name + '_storage_detaching', StorageDetachingEvent) - - for action_name in self.framework.meta.actions: - action_name = action_name.replace('-', '_') - self.on.define_event(action_name + '_action', ActionEvent) - - @property - def app(self) -> model.Application: - """Application that this unit is part of.""" - return self.framework.model.app - - @property - def unit(self) -> model.Unit: - """Unit that this execution is responsible for.""" - return self.framework.model.unit - - @property - def meta(self) -> 'CharmMeta': - """CharmMeta of this charm. - """ - return self.framework.meta - - @property - def charm_dir(self) -> pathlib.Path: - """Root directory of the Charm as it is running. - """ - return self.framework.charm_dir - - -class CharmMeta: - """Object containing the metadata for the charm. - - This is read from metadata.yaml and/or actions.yaml. Generally charms will - define this information, rather than reading it at runtime. This class is - mostly for the framework to understand what the charm has defined. - - The maintainers, tags, terms, series, and extra_bindings attributes are all - lists of strings. The requires, provides, peers, relations, storage, - resources, and payloads attributes are all mappings of names to instances - of the respective RelationMeta, StorageMeta, ResourceMeta, or PayloadMeta. - - The relations attribute is a convenience accessor which includes all of the - requires, provides, and peers RelationMeta items. If needed, the role of - the relation definition can be obtained from its role attribute. - - Attributes: - name: The name of this charm - summary: Short description of what this charm does - description: Long description for this charm - maintainers: A list of strings of the email addresses of the maintainers - of this charm. - tags: Charm store tag metadata for categories associated with this charm. - terms: Charm store terms that should be agreed to before this charm can - be deployed. (Used for things like licensing issues.) - series: The list of supported OS series that this charm can support. - The first entry in the list is the default series that will be - used by deploy if no other series is requested by the user. - subordinate: True/False whether this charm is intended to be used as a - subordinate charm. - min_juju_version: If supplied, indicates this charm needs features that - are not available in older versions of Juju. - requires: A dict of {name: :class:`RelationMeta` } for each 'requires' relation. - provides: A dict of {name: :class:`RelationMeta` } for each 'provides' relation. - peers: A dict of {name: :class:`RelationMeta` } for each 'peer' relation. - relations: A dict containing all :class:`RelationMeta` attributes (merged from other - sections) - storages: A dict of {name: :class:`StorageMeta`} for each defined storage. - resources: A dict of {name: :class:`ResourceMeta`} for each defined resource. - payloads: A dict of {name: :class:`PayloadMeta`} for each defined payload. - extra_bindings: A dict of additional named bindings that a charm can use - for network configuration. - actions: A dict of {name: :class:`ActionMeta`} for actions that the charm has defined. - Args: - raw: a mapping containing the contents of metadata.yaml - actions_raw: a mapping containing the contents of actions.yaml - """ - - def __init__(self, raw: dict = {}, actions_raw: dict = {}): - self.name = raw.get('name', '') - self.summary = raw.get('summary', '') - self.description = raw.get('description', '') - self.maintainers = [] - if 'maintainer' in raw: - self.maintainers.append(raw['maintainer']) - if 'maintainers' in raw: - self.maintainers.extend(raw['maintainers']) - self.tags = raw.get('tags', []) - self.terms = raw.get('terms', []) - self.series = raw.get('series', []) - self.subordinate = raw.get('subordinate', False) - self.min_juju_version = raw.get('min-juju-version') - self.requires = {name: RelationMeta('requires', name, rel) - for name, rel in raw.get('requires', {}).items()} - self.provides = {name: RelationMeta('provides', name, rel) - for name, rel in raw.get('provides', {}).items()} - # TODO: (jam 2020-05-11) The *role* should be 'peer' even though it comes from the - # 'peers' section. - self.peers = {name: RelationMeta('peers', name, rel) - for name, rel in raw.get('peers', {}).items()} - self.relations = {} - self.relations.update(self.requires) - self.relations.update(self.provides) - self.relations.update(self.peers) - self.storages = {name: StorageMeta(name, storage) - for name, storage in raw.get('storage', {}).items()} - self.resources = {name: ResourceMeta(name, res) - for name, res in raw.get('resources', {}).items()} - self.payloads = {name: PayloadMeta(name, payload) - for name, payload in raw.get('payloads', {}).items()} - self.extra_bindings = raw.get('extra-bindings', {}) - self.actions = {name: ActionMeta(name, action) for name, action in actions_raw.items()} - - @classmethod - def from_yaml( - cls, metadata: typing.Union[str, typing.TextIO], - actions: typing.Optional[typing.Union[str, typing.TextIO]] = None): - """Instantiate a CharmMeta from a YAML description of metadata.yaml. - - Args: - metadata: A YAML description of charm metadata (name, relations, etc.) - This can be a simple string, or a file-like object. (passed to `yaml.safe_load`). - actions: YAML description of Actions for this charm (eg actions.yaml) - """ - meta = yaml.safe_load(metadata) - raw_actions = {} - if actions is not None: - raw_actions = yaml.safe_load(actions) - return cls(meta, raw_actions) - - -class RelationMeta: - """Object containing metadata about a relation definition. - - Should not be constructed directly by Charm code. Is gotten from one of - :attr:`CharmMeta.peers`, :attr:`CharmMeta.requires`, :attr:`CharmMeta.provides`, - :attr:`CharmMeta.relations`. - - Attributes: - role: This is one of requires/provides/peers - relation_name: Name of this relation from metadata.yaml - interface_name: Optional definition of the interface protocol. - scope: "global" or "container" scope based on how the relation should be used. - """ - - def __init__(self, role, relation_name, raw): - self.role = role - self.relation_name = relation_name - self.interface_name = raw['interface'] - self.scope = raw.get('scope') - - -class StorageMeta: - """Object containing metadata about a storage definition.""" - - def __init__(self, name, raw): - self.storage_name = name - self.type = raw['type'] - self.description = raw.get('description', '') - self.shared = raw.get('shared', False) - self.read_only = raw.get('read-only', False) - self.minimum_size = raw.get('minimum-size') - self.location = raw.get('location') - self.multiple_range = None - if 'multiple' in raw: - range = raw['multiple']['range'] - if '-' not in range: - self.multiple_range = (int(range), int(range)) - else: - range = range.split('-') - self.multiple_range = (int(range[0]), int(range[1]) if range[1] else None) - - -class ResourceMeta: - """Object containing metadata about a resource definition.""" - - def __init__(self, name, raw): - self.resource_name = name - self.type = raw['type'] - self.filename = raw.get('filename', None) - self.description = raw.get('description', '') - - -class PayloadMeta: - """Object containing metadata about a payload definition.""" - - def __init__(self, name, raw): - self.payload_name = name - self.type = raw['type'] - - -class ActionMeta: - """Object containing metadata about an action's definition.""" - - def __init__(self, name, raw=None): - raw = raw or {} - self.name = name - self.title = raw.get('title', '') - self.description = raw.get('description', '') - self.parameters = raw.get('params', {}) # {: } - self.required = raw.get('required', []) # [, ...] diff --git a/magma/squid_cnf/charms/squid/mod/operator/ops/framework.py b/magma/squid_cnf/charms/squid/mod/operator/ops/framework.py deleted file mode 100755 index 51d46ba16886bfffce0fe7b9ad91f3ac0b5902a4..0000000000000000000000000000000000000000 --- a/magma/squid_cnf/charms/squid/mod/operator/ops/framework.py +++ /dev/null @@ -1,1134 +0,0 @@ -# Copyright 2019-2020 Canonical Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import collections -import collections.abc -import inspect -import keyword -import marshal -import os -import pdb -import pickle -import re -import sqlite3 -import sys -import types -import weakref -from datetime import timedelta - -from ops import charm - - -class Handle: - """Handle defines a name for an object in the form of a hierarchical path. - - The provided parent is the object (or that object's handle) that this handle - sits under, or None if the object identified by this handle stands by itself - as the root of its own hierarchy. - - The handle kind is a string that defines a namespace so objects with the - same parent and kind will have unique keys. - - The handle key is a string uniquely identifying the object. No other objects - under the same parent and kind may have the same key. - """ - - def __init__(self, parent, kind, key): - if parent and not isinstance(parent, Handle): - parent = parent.handle - self._parent = parent - self._kind = kind - self._key = key - if parent: - if key: - self._path = "{}/{}[{}]".format(parent, kind, key) - else: - self._path = "{}/{}".format(parent, kind) - else: - if key: - self._path = "{}[{}]".format(kind, key) - else: - self._path = "{}".format(kind) - - def nest(self, kind, key): - return Handle(self, kind, key) - - def __hash__(self): - return hash((self.parent, self.kind, self.key)) - - def __eq__(self, other): - return (self.parent, self.kind, self.key) == (other.parent, other.kind, other.key) - - def __str__(self): - return self.path - - @property - def parent(self): - return self._parent - - @property - def kind(self): - return self._kind - - @property - def key(self): - return self._key - - @property - def path(self): - return self._path - - @classmethod - def from_path(cls, path): - handle = None - for pair in path.split("/"): - pair = pair.split("[") - good = False - if len(pair) == 1: - kind, key = pair[0], None - good = True - elif len(pair) == 2: - kind, key = pair - if key and key[-1] == ']': - key = key[:-1] - good = True - if not good: - raise RuntimeError("attempted to restore invalid handle path {}".format(path)) - handle = Handle(handle, kind, key) - return handle - - -class EventBase: - - def __init__(self, handle): - self.handle = handle - self.deferred = False - - def defer(self): - self.deferred = True - - def snapshot(self): - """Return the snapshot data that should be persisted. - - Subclasses must override to save any custom state. - """ - return None - - def restore(self, snapshot): - """Restore the value state from the given snapshot. - - Subclasses must override to restore their custom state. - """ - self.deferred = False - - -class EventSource: - """EventSource wraps an event type with a descriptor to facilitate observing and emitting. - - It is generally used as: - - class SomethingHappened(EventBase): - pass - - class SomeObject(Object): - something_happened = EventSource(SomethingHappened) - - With that, instances of that type will offer the someobj.something_happened - attribute which is a BoundEvent and may be used to emit and observe the event. - """ - - def __init__(self, event_type): - if not isinstance(event_type, type) or not issubclass(event_type, EventBase): - raise RuntimeError( - 'Event requires a subclass of EventBase as an argument, got {}'.format(event_type)) - self.event_type = event_type - self.event_kind = None - self.emitter_type = None - - def _set_name(self, emitter_type, event_kind): - if self.event_kind is not None: - raise RuntimeError( - 'EventSource({}) reused as {}.{} and {}.{}'.format( - self.event_type.__name__, - self.emitter_type.__name__, - self.event_kind, - emitter_type.__name__, - event_kind, - )) - self.event_kind = event_kind - self.emitter_type = emitter_type - - def __get__(self, emitter, emitter_type=None): - if emitter is None: - return self - # Framework might not be available if accessed as CharmClass.on.event - # rather than charm_instance.on.event, but in that case it couldn't be - # emitted anyway, so there's no point to registering it. - framework = getattr(emitter, 'framework', None) - if framework is not None: - framework.register_type(self.event_type, emitter, self.event_kind) - return BoundEvent(emitter, self.event_type, self.event_kind) - - -class BoundEvent: - - def __repr__(self): - return ''.format( - self.event_type.__name__, - type(self.emitter).__name__, - self.event_kind, - hex(id(self)), - ) - - def __init__(self, emitter, event_type, event_kind): - self.emitter = emitter - self.event_type = event_type - self.event_kind = event_kind - - def emit(self, *args, **kwargs): - """Emit event to all registered observers. - - The current storage state is committed before and after each observer is notified. - """ - framework = self.emitter.framework - key = framework._next_event_key() - event = self.event_type(Handle(self.emitter, self.event_kind, key), *args, **kwargs) - framework._emit(event) - - -class HandleKind: - """Helper descriptor to define the Object.handle_kind field. - - The handle_kind for an object defaults to its type name, but it may - be explicitly overridden if desired. - """ - - def __get__(self, obj, obj_type): - kind = obj_type.__dict__.get("handle_kind") - if kind: - return kind - return obj_type.__name__ - - -class _Metaclass(type): - """Helper class to ensure proper instantiation of Object-derived classes. - - This class currently has a single purpose: events derived from EventSource - that are class attributes of Object-derived classes need to be told what - their name is in that class. For example, in - - class SomeObject(Object): - something_happened = EventSource(SomethingHappened) - - the instance of EventSource needs to know it's called 'something_happened'. - - Starting from python 3.6 we could use __set_name__ on EventSource for this, - but until then this (meta)class does the equivalent work. - - TODO: when we drop support for 3.5 drop this class, and rename _set_name in - EventSource to __set_name__; everything should continue to work. - - """ - - def __new__(typ, *a, **kw): - k = super().__new__(typ, *a, **kw) - # k is now the Object-derived class; loop over its class attributes - for n, v in vars(k).items(): - # we could do duck typing here if we want to support - # non-EventSource-derived shenanigans. We don't. - if isinstance(v, EventSource): - # this is what 3.6+ does automatically for us: - v._set_name(k, n) - return k - - -class Object(metaclass=_Metaclass): - - handle_kind = HandleKind() - - def __init__(self, parent, key): - kind = self.handle_kind - if isinstance(parent, Framework): - self.framework = parent - # Avoid Framework instances having a circular reference to themselves. - if self.framework is self: - self.framework = weakref.proxy(self.framework) - self.handle = Handle(None, kind, key) - else: - self.framework = parent.framework - self.handle = Handle(parent, kind, key) - self.framework._track(self) - - # TODO Detect conflicting handles here. - - @property - def model(self): - return self.framework.model - - -class ObjectEvents(Object): - """Convenience type to allow defining .on attributes at class level.""" - - handle_kind = "on" - - def __init__(self, parent=None, key=None): - if parent is not None: - super().__init__(parent, key) - else: - self._cache = weakref.WeakKeyDictionary() - - def __get__(self, emitter, emitter_type): - if emitter is None: - return self - instance = self._cache.get(emitter) - if instance is None: - # Same type, different instance, more data. Doing this unusual construct - # means people can subclass just this one class to have their own 'on'. - instance = self._cache[emitter] = type(self)(emitter) - return instance - - @classmethod - def define_event(cls, event_kind, event_type): - """Define an event on this type at runtime. - - cls: a type to define an event on. - - event_kind: an attribute name that will be used to access the - event. Must be a valid python identifier, not be a keyword - or an existing attribute. - - event_type: a type of the event to define. - - """ - prefix = 'unable to define an event with event_kind that ' - if not event_kind.isidentifier(): - raise RuntimeError(prefix + 'is not a valid python identifier: ' + event_kind) - elif keyword.iskeyword(event_kind): - raise RuntimeError(prefix + 'is a python keyword: ' + event_kind) - try: - getattr(cls, event_kind) - raise RuntimeError( - prefix + 'overlaps with an existing type {} attribute: {}'.format(cls, event_kind)) - except AttributeError: - pass - - event_descriptor = EventSource(event_type) - event_descriptor._set_name(cls, event_kind) - setattr(cls, event_kind, event_descriptor) - - def events(self): - """Return a mapping of event_kinds to bound_events for all available events. - """ - events_map = {} - # We have to iterate over the class rather than instance to allow for properties which - # might call this method (e.g., event views), leading to infinite recursion. - for attr_name, attr_value in inspect.getmembers(type(self)): - if isinstance(attr_value, EventSource): - # We actually care about the bound_event, however, since it - # provides the most info for users of this method. - event_kind = attr_name - bound_event = getattr(self, event_kind) - events_map[event_kind] = bound_event - return events_map - - def __getitem__(self, key): - return PrefixedEvents(self, key) - - -class PrefixedEvents: - - def __init__(self, emitter, key): - self._emitter = emitter - self._prefix = key.replace("-", "_") + '_' - - def __getattr__(self, name): - return getattr(self._emitter, self._prefix + name) - - -class PreCommitEvent(EventBase): - pass - - -class CommitEvent(EventBase): - pass - - -class FrameworkEvents(ObjectEvents): - pre_commit = EventSource(PreCommitEvent) - commit = EventSource(CommitEvent) - - -class NoSnapshotError(Exception): - - def __init__(self, handle_path): - self.handle_path = handle_path - - def __str__(self): - return 'no snapshot data found for {} object'.format(self.handle_path) - - -class NoTypeError(Exception): - - def __init__(self, handle_path): - self.handle_path = handle_path - - def __str__(self): - return "cannot restore {} since no class was registered for it".format(self.handle_path) - - -class SQLiteStorage: - - DB_LOCK_TIMEOUT = timedelta(hours=1) - - def __init__(self, filename): - # The isolation_level argument is set to None such that the implicit - # transaction management behavior of the sqlite3 module is disabled. - self._db = sqlite3.connect(str(filename), - isolation_level=None, - timeout=self.DB_LOCK_TIMEOUT.total_seconds()) - self._setup() - - def _setup(self): - # Make sure that the database is locked until the connection is closed, - # not until the transaction ends. - self._db.execute("PRAGMA locking_mode=EXCLUSIVE") - c = self._db.execute("BEGIN") - c.execute("SELECT count(name) FROM sqlite_master WHERE type='table' AND name='snapshot'") - if c.fetchone()[0] == 0: - # Keep in mind what might happen if the process dies somewhere below. - # The system must not be rendered permanently broken by that. - self._db.execute("CREATE TABLE snapshot (handle TEXT PRIMARY KEY, data BLOB)") - self._db.execute(''' - CREATE TABLE notice ( - sequence INTEGER PRIMARY KEY AUTOINCREMENT, - event_path TEXT, - observer_path TEXT, - method_name TEXT) - ''') - self._db.commit() - - def close(self): - self._db.close() - - def commit(self): - self._db.commit() - - # There's commit but no rollback. For abort to be supported, we'll need logic that - # can rollback decisions made by third-party code in terms of the internal state - # of objects that have been snapshotted, and hooks to let them know about it and - # take the needed actions to undo their logic until the last snapshot. - # This is doable but will increase significantly the chances for mistakes. - - def save_snapshot(self, handle_path, snapshot_data): - self._db.execute("REPLACE INTO snapshot VALUES (?, ?)", (handle_path, snapshot_data)) - - def load_snapshot(self, handle_path): - c = self._db.cursor() - c.execute("SELECT data FROM snapshot WHERE handle=?", (handle_path,)) - row = c.fetchone() - if row: - return row[0] - return None - - def drop_snapshot(self, handle_path): - self._db.execute("DELETE FROM snapshot WHERE handle=?", (handle_path,)) - - def save_notice(self, event_path, observer_path, method_name): - self._db.execute('INSERT INTO notice VALUES (NULL, ?, ?, ?)', - (event_path, observer_path, method_name)) - - def drop_notice(self, event_path, observer_path, method_name): - self._db.execute(''' - DELETE FROM notice - WHERE event_path=? - AND observer_path=? - AND method_name=? - ''', (event_path, observer_path, method_name)) - - def notices(self, event_path): - if event_path: - c = self._db.execute(''' - SELECT event_path, observer_path, method_name - FROM notice - WHERE event_path=? - ORDER BY sequence - ''', (event_path,)) - else: - c = self._db.execute(''' - SELECT event_path, observer_path, method_name - FROM notice - ORDER BY sequence - ''') - while True: - rows = c.fetchmany() - if not rows: - break - for row in rows: - yield tuple(row) - - -# the message to show to the user when a pdb breakpoint goes active -_BREAKPOINT_WELCOME_MESSAGE = """ -Starting pdb to debug charm operator. -Run `h` for help, `c` to continue, or `exit`/CTRL-d to abort. -Future breakpoints may interrupt execution again. -More details at https://discourse.jujucharms.com/t/debugging-charm-hooks - -""" - - -class Framework(Object): - - on = FrameworkEvents() - - # Override properties from Object so that we can set them in __init__. - model = None - meta = None - charm_dir = None - - def __init__(self, data_path, charm_dir, meta, model): - - super().__init__(self, None) - - self._data_path = data_path - self.charm_dir = charm_dir - self.meta = meta - self.model = model - self._observers = [] # [(observer_path, method_name, parent_path, event_key)] - self._observer = weakref.WeakValueDictionary() # {observer_path: observer} - self._objects = weakref.WeakValueDictionary() - self._type_registry = {} # {(parent_path, kind): cls} - self._type_known = set() # {cls} - - self._storage = SQLiteStorage(data_path) - - # We can't use the higher-level StoredState because it relies on events. - self.register_type(StoredStateData, None, StoredStateData.handle_kind) - stored_handle = Handle(None, StoredStateData.handle_kind, '_stored') - try: - self._stored = self.load_snapshot(stored_handle) - except NoSnapshotError: - self._stored = StoredStateData(self, '_stored') - self._stored['event_count'] = 0 - - # Hook into builtin breakpoint, so if Python >= 3.7, devs will be able to just do - # breakpoint(); if Python < 3.7, this doesn't affect anything - sys.breakpointhook = self.breakpoint - - # Flag to indicate that we already presented the welcome message in a debugger breakpoint - self._breakpoint_welcomed = False - - # Parse once the env var, which may be used multiple times later - debug_at = os.environ.get('JUJU_DEBUG_AT') - self._juju_debug_at = debug_at.split(',') if debug_at else () - - def close(self): - self._storage.close() - - def _track(self, obj): - """Track object and ensure it is the only object created using its handle path.""" - if obj is self: - # Framework objects don't track themselves - return - if obj.handle.path in self.framework._objects: - raise RuntimeError( - 'two objects claiming to be {} have been created'.format(obj.handle.path)) - self._objects[obj.handle.path] = obj - - def _forget(self, obj): - """Stop tracking the given object. See also _track.""" - self._objects.pop(obj.handle.path, None) - - def commit(self): - # Give a chance for objects to persist data they want to before a commit is made. - self.on.pre_commit.emit() - # Make sure snapshots are saved by instances of StoredStateData. Any possible state - # modifications in on_commit handlers of instances of other classes will not be persisted. - self.on.commit.emit() - # Save our event count after all events have been emitted. - self.save_snapshot(self._stored) - self._storage.commit() - - def register_type(self, cls, parent, kind=None): - if parent and not isinstance(parent, Handle): - parent = parent.handle - if parent: - parent_path = parent.path - else: - parent_path = None - if not kind: - kind = cls.handle_kind - self._type_registry[(parent_path, kind)] = cls - self._type_known.add(cls) - - def save_snapshot(self, value): - """Save a persistent snapshot of the provided value. - - The provided value must implement the following interface: - - value.handle = Handle(...) - value.snapshot() => {...} # Simple builtin types only. - value.restore(snapshot) # Restore custom state from prior snapshot. - """ - if type(value) not in self._type_known: - raise RuntimeError( - 'cannot save {} values before registering that type'.format(type(value).__name__)) - data = value.snapshot() - - # Use marshal as a validator, enforcing the use of simple types, as we later the - # information is really pickled, which is too error prone for future evolution of the - # stored data (e.g. if the developer stores a custom object and later changes its - # class name; when unpickling the original class will not be there and event - # data loading will fail). - try: - marshal.dumps(data) - except ValueError: - msg = "unable to save the data for {}, it must contain only simple types: {!r}" - raise ValueError(msg.format(value.__class__.__name__, data)) - - # Use pickle for serialization, so the value remains portable. - raw_data = pickle.dumps(data) - self._storage.save_snapshot(value.handle.path, raw_data) - - def load_snapshot(self, handle): - parent_path = None - if handle.parent: - parent_path = handle.parent.path - cls = self._type_registry.get((parent_path, handle.kind)) - if not cls: - raise NoTypeError(handle.path) - raw_data = self._storage.load_snapshot(handle.path) - if not raw_data: - raise NoSnapshotError(handle.path) - data = pickle.loads(raw_data) - obj = cls.__new__(cls) - obj.framework = self - obj.handle = handle - obj.restore(data) - self._track(obj) - return obj - - def drop_snapshot(self, handle): - self._storage.drop_snapshot(handle.path) - - def observe(self, bound_event, observer): - """Register observer to be called when bound_event is emitted. - - The bound_event is generally provided as an attribute of the object that emits - the event, and is created in this style: - - class SomeObject: - something_happened = Event(SomethingHappened) - - That event may be observed as: - - framework.observe(someobj.something_happened, self.on_something_happened) - - If the method to be called follows the name convention "on_", it - may be omitted from the observe call. That means the above is equivalent to: - - framework.observe(someobj.something_happened, self) - - """ - if not isinstance(bound_event, BoundEvent): - raise RuntimeError( - 'Framework.observe requires a BoundEvent as second parameter, got {}'.format( - bound_event)) - - event_type = bound_event.event_type - event_kind = bound_event.event_kind - emitter = bound_event.emitter - - self.register_type(event_type, emitter, event_kind) - - if hasattr(emitter, "handle"): - emitter_path = emitter.handle.path - else: - raise RuntimeError( - 'event emitter {} must have a "handle" attribute'.format(type(emitter).__name__)) - - method_name = None - if isinstance(observer, types.MethodType): - method_name = observer.__name__ - observer = observer.__self__ - else: - method_name = "on_" + event_kind - if not hasattr(observer, method_name): - raise RuntimeError( - 'Observer method not provided explicitly' - ' and {} type has no "{}" method'.format(type(observer).__name__, - method_name)) - - # Validate that the method has an acceptable call signature. - sig = inspect.signature(getattr(observer, method_name)) - # Self isn't included in the params list, so the first arg will be the event. - extra_params = list(sig.parameters.values())[1:] - if not sig.parameters: - raise TypeError( - '{}.{} must accept event parameter'.format(type(observer).__name__, method_name)) - elif any(param.default is inspect.Parameter.empty for param in extra_params): - # Allow for additional optional params, since there's no reason to exclude them, but - # required params will break. - raise TypeError( - '{}.{} has extra required parameter'.format(type(observer).__name__, method_name)) - - # TODO Prevent the exact same parameters from being registered more than once. - - self._observer[observer.handle.path] = observer - self._observers.append((observer.handle.path, method_name, emitter_path, event_kind)) - - def _next_event_key(self): - """Return the next event key that should be used, incrementing the internal counter.""" - # Increment the count first; this means the keys will start at 1, and 0 - # means no events have been emitted. - self._stored['event_count'] += 1 - return str(self._stored['event_count']) - - def _emit(self, event): - """See BoundEvent.emit for the public way to call this.""" - - # Save the event for all known observers before the first notification - # takes place, so that either everyone interested sees it, or nobody does. - self.save_snapshot(event) - event_path = event.handle.path - event_kind = event.handle.kind - parent_path = event.handle.parent.path - # TODO Track observers by (parent_path, event_kind) rather than as a list of - # all observers. Avoiding linear search through all observers for every event - for observer_path, method_name, _parent_path, _event_kind in self._observers: - if _parent_path != parent_path: - continue - if _event_kind and _event_kind != event_kind: - continue - # Again, only commit this after all notices are saved. - self._storage.save_notice(event_path, observer_path, method_name) - self._reemit(event_path) - - def reemit(self): - """Reemit previously deferred events to the observers that deferred them. - - Only the specific observers that have previously deferred the event will be - notified again. Observers that asked to be notified about events after it's - been first emitted won't be notified, as that would mean potentially observing - events out of order. - """ - self._reemit() - - def _reemit(self, single_event_path=None): - last_event_path = None - deferred = True - for event_path, observer_path, method_name in self._storage.notices(single_event_path): - event_handle = Handle.from_path(event_path) - - if last_event_path != event_path: - if not deferred: - self._storage.drop_snapshot(last_event_path) - last_event_path = event_path - deferred = False - - try: - event = self.load_snapshot(event_handle) - except NoTypeError: - self._storage.drop_notice(event_path, observer_path, method_name) - continue - - event.deferred = False - observer = self._observer.get(observer_path) - if observer: - custom_handler = getattr(observer, method_name, None) - if custom_handler: - event_is_from_juju = isinstance(event, charm.HookEvent) - event_is_action = isinstance(event, charm.ActionEvent) - if (event_is_from_juju or event_is_action) and 'hook' in self._juju_debug_at: - # Present the welcome message and run under PDB. - self._show_debug_code_message() - pdb.runcall(custom_handler, event) - else: - # Regular call to the registered method. - custom_handler(event) - - if event.deferred: - deferred = True - else: - self._storage.drop_notice(event_path, observer_path, method_name) - # We intentionally consider this event to be dead and reload it from - # scratch in the next path. - self.framework._forget(event) - - if not deferred: - self._storage.drop_snapshot(last_event_path) - - def _show_debug_code_message(self): - """Present the welcome message (only once!) when using debugger functionality.""" - if not self._breakpoint_welcomed: - self._breakpoint_welcomed = True - print(_BREAKPOINT_WELCOME_MESSAGE, file=sys.stderr, end='') - - def breakpoint(self, name=None): - """Add breakpoint, optionally named, at the place where this method is called. - - For the breakpoint to be activated the JUJU_DEBUG_AT environment variable - must be set to "all" or to the specific name parameter provided, if any. In every - other situation calling this method does nothing. - - The framework also provides a standard breakpoint named "hook", that will - stop execution when a hook event is about to be handled. - - For those reasons, the "all" and "hook" breakpoint names are reserved. - """ - # If given, validate the name comply with all the rules - if name is not None: - if not isinstance(name, str): - raise TypeError('breakpoint names must be strings') - if name in ('hook', 'all'): - raise ValueError('breakpoint names "all" and "hook" are reserved') - if not re.match(r'^[a-z0-9]([a-z0-9\-]*[a-z0-9])?$', name): - raise ValueError('breakpoint names must look like "foo" or "foo-bar"') - - indicated_breakpoints = self._juju_debug_at - if 'all' in indicated_breakpoints or name in indicated_breakpoints: - self._show_debug_code_message() - - # If we call set_trace() directly it will open the debugger *here*, so indicating - # it to use our caller's frame - code_frame = inspect.currentframe().f_back - pdb.Pdb().set_trace(code_frame) - - -class StoredStateData(Object): - - def __init__(self, parent, attr_name): - super().__init__(parent, attr_name) - self._cache = {} - self.dirty = False - - def __getitem__(self, key): - return self._cache.get(key) - - def __setitem__(self, key, value): - self._cache[key] = value - self.dirty = True - - def __contains__(self, key): - return key in self._cache - - def snapshot(self): - return self._cache - - def restore(self, snapshot): - self._cache = snapshot - self.dirty = False - - def on_commit(self, event): - if self.dirty: - self.framework.save_snapshot(self) - self.dirty = False - - -class BoundStoredState: - - def __init__(self, parent, attr_name): - parent.framework.register_type(StoredStateData, parent) - - handle = Handle(parent, StoredStateData.handle_kind, attr_name) - try: - data = parent.framework.load_snapshot(handle) - except NoSnapshotError: - data = StoredStateData(parent, attr_name) - - # __dict__ is used to avoid infinite recursion. - self.__dict__["_data"] = data - self.__dict__["_attr_name"] = attr_name - - parent.framework.observe(parent.framework.on.commit, self._data) - - def __getattr__(self, key): - # "on" is the only reserved key that can't be used in the data map. - if key == "on": - return self._data.on - if key not in self._data: - raise AttributeError("attribute '{}' is not stored".format(key)) - return _wrap_stored(self._data, self._data[key]) - - def __setattr__(self, key, value): - if key == "on": - raise AttributeError("attribute 'on' is reserved and cannot be set") - - value = _unwrap_stored(self._data, value) - - if not isinstance(value, (type(None), int, float, str, bytes, list, dict, set)): - raise AttributeError( - 'attribute {!r} cannot be a {}: must be int/float/dict/list/etc'.format( - key, type(value).__name__)) - - self._data[key] = _unwrap_stored(self._data, value) - - def set_default(self, **kwargs): - """"Set the value of any given key if it has not already been set""" - for k, v in kwargs.items(): - if k not in self._data: - self._data[k] = v - - -class StoredState: - """A class used to store data the charm needs persisted across invocations. - - Example:: - - class MyClass(Object): - _stored = StoredState() - - Instances of `MyClass` can transparently save state between invocations by - setting attributes on `_stored`. Initial state should be set with - `set_default` on the bound object, that is:: - - class MyClass(Object): - _stored = StoredState() - - def __init__(self, parent, key): - super().__init__(parent, key) - self._stored.set_default(seen=set()) - self.framework.observe(self.on.seen, self._on_seen) - - def _on_seen(self, event): - self._stored.seen.add(event.uuid) - - """ - - def __init__(self): - self.parent_type = None - self.attr_name = None - - def __get__(self, parent, parent_type=None): - if self.parent_type is not None and self.parent_type not in parent_type.mro(): - # the StoredState instance is being shared between two unrelated classes - # -> unclear what is exepcted of us -> bail out - raise RuntimeError( - 'StoredState shared by {} and {}'.format( - self.parent_type.__name__, parent_type.__name__)) - - if parent is None: - # accessing via the class directly (e.g. MyClass.stored) - return self - - bound = None - if self.attr_name is not None: - bound = parent.__dict__.get(self.attr_name) - if bound is not None: - # we already have the thing from a previous pass, huzzah - return bound - - # need to find ourselves amongst the parent's bases - for cls in parent_type.mro(): - for attr_name, attr_value in cls.__dict__.items(): - if attr_value is not self: - continue - # we've found ourselves! is it the first time? - if bound is not None: - # the StoredState instance is being stored in two different - # attributes -> unclear what is expected of us -> bail out - raise RuntimeError("StoredState shared by {0}.{1} and {0}.{2}".format( - cls.__name__, self.attr_name, attr_name)) - # we've found ourselves for the first time; save where, and bind the object - self.attr_name = attr_name - self.parent_type = cls - bound = BoundStoredState(parent, attr_name) - - if bound is not None: - # cache the bound object to avoid the expensive lookup the next time - # (don't use setattr, to keep things symmetric with the fast-path lookup above) - parent.__dict__[self.attr_name] = bound - return bound - - raise AttributeError( - 'cannot find {} attribute in type {}'.format( - self.__class__.__name__, parent_type.__name__)) - - -def _wrap_stored(parent_data, value): - t = type(value) - if t is dict: - return StoredDict(parent_data, value) - if t is list: - return StoredList(parent_data, value) - if t is set: - return StoredSet(parent_data, value) - return value - - -def _unwrap_stored(parent_data, value): - t = type(value) - if t is StoredDict or t is StoredList or t is StoredSet: - return value._under - return value - - -class StoredDict(collections.abc.MutableMapping): - - def __init__(self, stored_data, under): - self._stored_data = stored_data - self._under = under - - def __getitem__(self, key): - return _wrap_stored(self._stored_data, self._under[key]) - - def __setitem__(self, key, value): - self._under[key] = _unwrap_stored(self._stored_data, value) - self._stored_data.dirty = True - - def __delitem__(self, key): - del self._under[key] - self._stored_data.dirty = True - - def __iter__(self): - return self._under.__iter__() - - def __len__(self): - return len(self._under) - - def __eq__(self, other): - if isinstance(other, StoredDict): - return self._under == other._under - elif isinstance(other, collections.abc.Mapping): - return self._under == other - else: - return NotImplemented - - -class StoredList(collections.abc.MutableSequence): - - def __init__(self, stored_data, under): - self._stored_data = stored_data - self._under = under - - def __getitem__(self, index): - return _wrap_stored(self._stored_data, self._under[index]) - - def __setitem__(self, index, value): - self._under[index] = _unwrap_stored(self._stored_data, value) - self._stored_data.dirty = True - - def __delitem__(self, index): - del self._under[index] - self._stored_data.dirty = True - - def __len__(self): - return len(self._under) - - def insert(self, index, value): - self._under.insert(index, value) - self._stored_data.dirty = True - - def append(self, value): - self._under.append(value) - self._stored_data.dirty = True - - def __eq__(self, other): - if isinstance(other, StoredList): - return self._under == other._under - elif isinstance(other, collections.abc.Sequence): - return self._under == other - else: - return NotImplemented - - def __lt__(self, other): - if isinstance(other, StoredList): - return self._under < other._under - elif isinstance(other, collections.abc.Sequence): - return self._under < other - else: - return NotImplemented - - def __le__(self, other): - if isinstance(other, StoredList): - return self._under <= other._under - elif isinstance(other, collections.abc.Sequence): - return self._under <= other - else: - return NotImplemented - - def __gt__(self, other): - if isinstance(other, StoredList): - return self._under > other._under - elif isinstance(other, collections.abc.Sequence): - return self._under > other - else: - return NotImplemented - - def __ge__(self, other): - if isinstance(other, StoredList): - return self._under >= other._under - elif isinstance(other, collections.abc.Sequence): - return self._under >= other - else: - return NotImplemented - - -class StoredSet(collections.abc.MutableSet): - - def __init__(self, stored_data, under): - self._stored_data = stored_data - self._under = under - - def add(self, key): - self._under.add(key) - self._stored_data.dirty = True - - def discard(self, key): - self._under.discard(key) - self._stored_data.dirty = True - - def __contains__(self, key): - return key in self._under - - def __iter__(self): - return self._under.__iter__() - - def __len__(self): - return len(self._under) - - @classmethod - def _from_iterable(cls, it): - """Construct an instance of the class from any iterable input. - - Per https://docs.python.org/3/library/collections.abc.html - if the Set mixin is being used in a class with a different constructor signature, - you will need to override _from_iterable() with a classmethod that can construct - new instances from an iterable argument. - """ - return set(it) - - def __le__(self, other): - if isinstance(other, StoredSet): - return self._under <= other._under - elif isinstance(other, collections.abc.Set): - return self._under <= other - else: - return NotImplemented - - def __ge__(self, other): - if isinstance(other, StoredSet): - return self._under >= other._under - elif isinstance(other, collections.abc.Set): - return self._under >= other - else: - return NotImplemented - - def __eq__(self, other): - if isinstance(other, StoredSet): - return self._under == other._under - elif isinstance(other, collections.abc.Set): - return self._under == other - else: - return NotImplemented diff --git a/magma/squid_cnf/charms/squid/mod/operator/ops/jujuversion.py b/magma/squid_cnf/charms/squid/mod/operator/ops/jujuversion.py deleted file mode 100755 index 4517886218c143f8c0249ac7285dc594976f9b01..0000000000000000000000000000000000000000 --- a/magma/squid_cnf/charms/squid/mod/operator/ops/jujuversion.py +++ /dev/null @@ -1,85 +0,0 @@ -# Copyright 2020 Canonical Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import re -from functools import total_ordering - - -@total_ordering -class JujuVersion: - - PATTERN = r'''^ - (?P\d{1,9})\.(?P\d{1,9}) # and numbers are always there - ((?:\.|-(?P[a-z]+))(?P\d{1,9}))? # sometimes with . or - - (\.(?P\d{1,9}))?$ # and sometimes with a number. - ''' - - def __init__(self, version): - m = re.match(self.PATTERN, version, re.VERBOSE) - if not m: - raise RuntimeError('"{}" is not a valid Juju version string'.format(version)) - - d = m.groupdict() - self.major = int(m.group('major')) - self.minor = int(m.group('minor')) - self.tag = d['tag'] or '' - self.patch = int(d['patch'] or 0) - self.build = int(d['build'] or 0) - - def __repr__(self): - if self.tag: - s = '{}.{}-{}{}'.format(self.major, self.minor, self.tag, self.patch) - else: - s = '{}.{}.{}'.format(self.major, self.minor, self.patch) - if self.build > 0: - s += '.{}'.format(self.build) - return s - - def __eq__(self, other): - if self is other: - return True - if isinstance(other, str): - other = type(self)(other) - elif not isinstance(other, JujuVersion): - raise RuntimeError('cannot compare Juju version "{}" with "{}"'.format(self, other)) - return ( - self.major == other.major - and self.minor == other.minor - and self.tag == other.tag - and self.build == other.build - and self.patch == other.patch) - - def __lt__(self, other): - if self is other: - return False - if isinstance(other, str): - other = type(self)(other) - elif not isinstance(other, JujuVersion): - raise RuntimeError('cannot compare Juju version "{}" with "{}"'.format(self, other)) - - if self.major != other.major: - return self.major < other.major - elif self.minor != other.minor: - return self.minor < other.minor - elif self.tag != other.tag: - if not self.tag: - return False - elif not other.tag: - return True - return self.tag < other.tag - elif self.patch != other.patch: - return self.patch < other.patch - elif self.build != other.build: - return self.build < other.build - return False diff --git a/magma/squid_cnf/charms/squid/mod/operator/ops/log.py b/magma/squid_cnf/charms/squid/mod/operator/ops/log.py deleted file mode 100644 index a3f76a375a98e23c718e47bcde5c33b49f4031c7..0000000000000000000000000000000000000000 --- a/magma/squid_cnf/charms/squid/mod/operator/ops/log.py +++ /dev/null @@ -1,47 +0,0 @@ -# Copyright 2020 Canonical Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import logging - - -class JujuLogHandler(logging.Handler): - """A handler for sending logs to Juju via juju-log.""" - - def __init__(self, model_backend, level=logging.DEBUG): - super().__init__(level) - self.model_backend = model_backend - - def emit(self, record): - self.model_backend.juju_log(record.levelname, self.format(record)) - - -def setup_root_logging(model_backend, debug=False): - """Setup python logging to forward messages to juju-log. - - By default, logging is set to DEBUG level, and messages will be filtered by Juju. - Charmers can also set their own default log level with:: - - logging.getLogger().setLevel(logging.INFO) - - model_backend -- a ModelBackend to use for juju-log - debug -- if True, write logs to stderr as well as to juju-log. - """ - logger = logging.getLogger() - logger.setLevel(logging.DEBUG) - logger.addHandler(JujuLogHandler(model_backend)) - if debug: - handler = logging.StreamHandler() - formatter = logging.Formatter('%(asctime)s %(levelname)-8s %(message)s') - handler.setFormatter(formatter) - logger.addHandler(handler) diff --git a/magma/squid_cnf/charms/squid/mod/operator/ops/main.py b/magma/squid_cnf/charms/squid/mod/operator/ops/main.py deleted file mode 100755 index 0f5391d76e45ba32dc652adfc99b2c7716d8af36..0000000000000000000000000000000000000000 --- a/magma/squid_cnf/charms/squid/mod/operator/ops/main.py +++ /dev/null @@ -1,235 +0,0 @@ -#!/usr/bin/env python3 -# Copyright 2019 Canonical Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -import subprocess -import sys -from pathlib import Path - -import yaml - -import ops.charm -import ops.framework -import ops.model -import logging - -from ops.log import setup_root_logging - -CHARM_STATE_FILE = '.unit-state.db' - - -logger = logging.getLogger() - - -def _get_charm_dir(): - charm_dir = os.environ.get("JUJU_CHARM_DIR") - if charm_dir is None: - # Assume $JUJU_CHARM_DIR/lib/op/main.py structure. - charm_dir = Path('{}/../../..'.format(__file__)).resolve() - else: - charm_dir = Path(charm_dir).resolve() - return charm_dir - - -def _load_metadata(charm_dir): - metadata = yaml.safe_load((charm_dir / 'metadata.yaml').read_text()) - - actions_meta = charm_dir / 'actions.yaml' - if actions_meta.exists(): - actions_metadata = yaml.safe_load(actions_meta.read_text()) - else: - actions_metadata = {} - return metadata, actions_metadata - - -def _create_event_link(charm, bound_event): - """Create a symlink for a particular event. - - charm -- A charm object. - bound_event -- An event for which to create a symlink. - """ - if issubclass(bound_event.event_type, ops.charm.HookEvent): - event_dir = charm.framework.charm_dir / 'hooks' - event_path = event_dir / bound_event.event_kind.replace('_', '-') - elif issubclass(bound_event.event_type, ops.charm.ActionEvent): - if not bound_event.event_kind.endswith("_action"): - raise RuntimeError( - 'action event name {} needs _action suffix'.format(bound_event.event_kind)) - event_dir = charm.framework.charm_dir / 'actions' - # The event_kind is suffixed with "_action" while the executable is not. - event_path = event_dir / bound_event.event_kind[:-len('_action')].replace('_', '-') - else: - raise RuntimeError( - 'cannot create a symlink: unsupported event type {}'.format(bound_event.event_type)) - - event_dir.mkdir(exist_ok=True) - if not event_path.exists(): - # CPython has different implementations for populating sys.argv[0] for Linux and Windows. - # For Windows it is always an absolute path (any symlinks are resolved) - # while for Linux it can be a relative path. - target_path = os.path.relpath(os.path.realpath(sys.argv[0]), str(event_dir)) - - # Ignore the non-symlink files or directories - # assuming the charm author knows what they are doing. - logger.debug( - 'Creating a new relative symlink at %s pointing to %s', - event_path, target_path) - event_path.symlink_to(target_path) - - -def _setup_event_links(charm_dir, charm): - """Set up links for supported events that originate from Juju. - - Whether a charm can handle an event or not can be determined by - introspecting which events are defined on it. - - Hooks or actions are created as symlinks to the charm code file - which is determined by inspecting symlinks provided by the charm - author at hooks/install or hooks/start. - - charm_dir -- A root directory of the charm. - charm -- An instance of the Charm class. - - """ - for bound_event in charm.on.events().values(): - # Only events that originate from Juju need symlinks. - if issubclass(bound_event.event_type, (ops.charm.HookEvent, ops.charm.ActionEvent)): - _create_event_link(charm, bound_event) - - -def _emit_charm_event(charm, event_name): - """Emits a charm event based on a Juju event name. - - charm -- A charm instance to emit an event from. - event_name -- A Juju event name to emit on a charm. - """ - event_to_emit = None - try: - event_to_emit = getattr(charm.on, event_name) - except AttributeError: - logger.debug("event %s not defined for %s", event_name, charm) - - # If the event is not supported by the charm implementation, do - # not error out or try to emit it. This is to support rollbacks. - if event_to_emit is not None: - args, kwargs = _get_event_args(charm, event_to_emit) - logger.debug('Emitting Juju event %s', event_name) - event_to_emit.emit(*args, **kwargs) - - -def _get_event_args(charm, bound_event): - event_type = bound_event.event_type - model = charm.framework.model - - if issubclass(event_type, ops.charm.RelationEvent): - relation_name = os.environ['JUJU_RELATION'] - relation_id = int(os.environ['JUJU_RELATION_ID'].split(':')[-1]) - relation = model.get_relation(relation_name, relation_id) - else: - relation = None - - remote_app_name = os.environ.get('JUJU_REMOTE_APP', '') - remote_unit_name = os.environ.get('JUJU_REMOTE_UNIT', '') - if remote_app_name or remote_unit_name: - if not remote_app_name: - if '/' not in remote_unit_name: - raise RuntimeError('invalid remote unit name: {}'.format(remote_unit_name)) - remote_app_name = remote_unit_name.split('/')[0] - args = [relation, model.get_app(remote_app_name)] - if remote_unit_name: - args.append(model.get_unit(remote_unit_name)) - return args, {} - elif relation: - return [relation], {} - return [], {} - - -def main(charm_class): - """Setup the charm and dispatch the observed event. - - The event name is based on the way this executable was called (argv[0]). - """ - charm_dir = _get_charm_dir() - - model_backend = ops.model.ModelBackend() - debug = ('JUJU_DEBUG' in os.environ) - setup_root_logging(model_backend, debug=debug) - - # Process the Juju event relevant to the current hook execution - # JUJU_HOOK_NAME, JUJU_FUNCTION_NAME, and JUJU_ACTION_NAME are not used - # in order to support simulation of events from debugging sessions. - # - # TODO: For Windows, when symlinks are used, this is not a valid - # method of getting an event name (see LP: #1854505). - juju_exec_path = Path(sys.argv[0]) - has_dispatch = juju_exec_path.name == 'dispatch' - if has_dispatch: - # The executable was 'dispatch', which means the actual hook we want to - # run needs to be looked up in the JUJU_DISPATCH_PATH env var, where it - # should be a path relative to the charm directory (the directory that - # holds `dispatch`). If that path actually exists, we want to run that - # before continuing. - dispatch_path = juju_exec_path.parent / Path(os.environ['JUJU_DISPATCH_PATH']) - if dispatch_path.exists() and dispatch_path.resolve() != juju_exec_path.resolve(): - argv = sys.argv.copy() - argv[0] = str(dispatch_path) - try: - subprocess.run(argv, check=True) - except subprocess.CalledProcessError as e: - logger.warning("hook %s exited with status %d", dispatch_path, e.returncode) - sys.exit(e.returncode) - juju_exec_path = dispatch_path - juju_event_name = juju_exec_path.name.replace('-', '_') - if juju_exec_path.parent.name == 'actions': - juju_event_name = '{}_action'.format(juju_event_name) - - metadata, actions_metadata = _load_metadata(charm_dir) - meta = ops.charm.CharmMeta(metadata, actions_metadata) - unit_name = os.environ['JUJU_UNIT_NAME'] - model = ops.model.Model(unit_name, meta, model_backend) - - # TODO: If Juju unit agent crashes after exit(0) from the charm code - # the framework will commit the snapshot but Juju will not commit its - # operation. - charm_state_path = charm_dir / CHARM_STATE_FILE - framework = ops.framework.Framework(charm_state_path, charm_dir, meta, model) - try: - charm = charm_class(framework, None) - - if not has_dispatch: - # When a charm is force-upgraded and a unit is in an error state Juju - # does not run upgrade-charm and instead runs the failed hook followed - # by config-changed. Given the nature of force-upgrading the hook setup - # code is not triggered on config-changed. - # - # 'start' event is included as Juju does not fire the install event for - # K8s charms (see LP: #1854635). - if (juju_event_name in ('install', 'start', 'upgrade_charm') - or juju_event_name.endswith('_storage_attached')): - _setup_event_links(charm_dir, charm) - - # TODO: Remove the collect_metrics check below as soon as the relevant - # Juju changes are made. - # - # Skip reemission of deferred events for collect-metrics events because - # they do not have the full access to all hook tools. - if juju_event_name != 'collect_metrics': - framework.reemit() - - _emit_charm_event(charm, juju_event_name) - - framework.commit() - finally: - framework.close() diff --git a/magma/squid_cnf/charms/squid/mod/operator/ops/model.py b/magma/squid_cnf/charms/squid/mod/operator/ops/model.py deleted file mode 100644 index 3d19ba8bff166aa0e3c0acc7dd4fe4b42690918c..0000000000000000000000000000000000000000 --- a/magma/squid_cnf/charms/squid/mod/operator/ops/model.py +++ /dev/null @@ -1,915 +0,0 @@ -# Copyright 2019 Canonical Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import json -import weakref -import os -import shutil -import tempfile -import time -import datetime -import re -import ipaddress -import decimal - -from abc import ABC, abstractmethod -from collections.abc import Mapping, MutableMapping -from pathlib import Path -from subprocess import run, PIPE, CalledProcessError - - -class Model: - - def __init__(self, unit_name, meta, backend): - self._cache = _ModelCache(backend) - self._backend = backend - self.unit = self.get_unit(unit_name) - self.app = self.unit.app - self.relations = RelationMapping(meta.relations, self.unit, self._backend, self._cache) - self.config = ConfigData(self._backend) - self.resources = Resources(list(meta.resources), self._backend) - self.pod = Pod(self._backend) - self.storages = StorageMapping(list(meta.storages), self._backend) - self._bindings = BindingMapping(self._backend) - - def get_unit(self, unit_name): - return self._cache.get(Unit, unit_name) - - def get_app(self, app_name): - return self._cache.get(Application, app_name) - - def get_relation(self, relation_name, relation_id=None): - """Get a specific Relation instance. - - If relation_id is given, this will return that Relation instance. - - If relation_id is not given, this will return the Relation instance if the - relation is established only once or None if it is not established. If this - same relation is established multiple times the error TooManyRelatedAppsError is raised. - """ - return self.relations._get_unique(relation_name, relation_id) - - def get_binding(self, binding_key): - """Get a network space binding. - - binding_key -- The relation name or instance to obtain bindings for. - - If binding_key is a relation name, the method returns the default binding for that - relation. If a relation instance is provided, the method first looks up a more specific - binding for that specific relation ID, and if none is found falls back to the default - binding for the relation name. - """ - return self._bindings.get(binding_key) - - -class _ModelCache: - - def __init__(self, backend): - self._backend = backend - self._weakrefs = weakref.WeakValueDictionary() - - def get(self, entity_type, *args): - key = (entity_type,) + args - entity = self._weakrefs.get(key) - if entity is None: - entity = entity_type(*args, backend=self._backend, cache=self) - self._weakrefs[key] = entity - return entity - - -class Application: - - def __init__(self, name, backend, cache): - self.name = name - self._backend = backend - self._cache = cache - self._is_our_app = self.name == self._backend.app_name - self._status = None - - @property - def status(self): - if not self._is_our_app: - return UnknownStatus() - - if not self._backend.is_leader(): - raise RuntimeError('cannot get application status as a non-leader unit') - - if self._status: - return self._status - - s = self._backend.status_get(is_app=True) - self._status = StatusBase.from_name(s['status'], s['message']) - return self._status - - @status.setter - def status(self, value): - if not isinstance(value, StatusBase): - raise InvalidStatusError( - 'invalid value provided for application {} status: {}'.format(self, value) - ) - - if not self._is_our_app: - raise RuntimeError('cannot to set status for a remote application {}'.format(self)) - - if not self._backend.is_leader(): - raise RuntimeError('cannot set application status as a non-leader unit') - - self._backend.status_set(value.name, value.message, is_app=True) - self._status = value - - def __repr__(self): - return '<{}.{} {}>'.format(type(self).__module__, type(self).__name__, self.name) - - -class Unit: - - def __init__(self, name, backend, cache): - self.name = name - - app_name = name.split('/')[0] - self.app = cache.get(Application, app_name) - - self._backend = backend - self._cache = cache - self._is_our_unit = self.name == self._backend.unit_name - self._status = None - - @property - def status(self): - if not self._is_our_unit: - return UnknownStatus() - - if self._status: - return self._status - - s = self._backend.status_get(is_app=False) - self._status = StatusBase.from_name(s['status'], s['message']) - return self._status - - @status.setter - def status(self, value): - if not isinstance(value, StatusBase): - raise InvalidStatusError( - 'invalid value provided for unit {} status: {}'.format(self, value) - ) - - if not self._is_our_unit: - raise RuntimeError('cannot set status for a remote unit {}'.format(self)) - - self._backend.status_set(value.name, value.message, is_app=False) - self._status = value - - def __repr__(self): - return '<{}.{} {}>'.format(type(self).__module__, type(self).__name__, self.name) - - def is_leader(self): - if self._is_our_unit: - # This value is not cached as it is not guaranteed to persist for the whole duration - # of a hook execution. - return self._backend.is_leader() - else: - raise RuntimeError( - 'cannot determine leadership status for remote applications: {}'.format(self) - ) - - def set_workload_version(self, version): - """Record the version of the software running as the workload. - - This shouldn't be confused with the revision of the charm. This is informative only; - shown in the output of 'juju status'. - """ - if not isinstance(version, str): - raise TypeError("workload version must be a str, not {}: {!r}".format( - type(version).__name__, version)) - self._backend.application_version_set(version) - - -class LazyMapping(Mapping, ABC): - - _lazy_data = None - - @abstractmethod - def _load(self): - raise NotImplementedError() - - @property - def _data(self): - data = self._lazy_data - if data is None: - data = self._lazy_data = self._load() - return data - - def _invalidate(self): - self._lazy_data = None - - def __contains__(self, key): - return key in self._data - - def __len__(self): - return len(self._data) - - def __iter__(self): - return iter(self._data) - - def __getitem__(self, key): - return self._data[key] - - -class RelationMapping(Mapping): - """Map of relation names to lists of Relation instances.""" - - def __init__(self, relations_meta, our_unit, backend, cache): - self._peers = set() - for name, relation_meta in relations_meta.items(): - if relation_meta.role == 'peers': - self._peers.add(name) - self._our_unit = our_unit - self._backend = backend - self._cache = cache - self._data = {relation_name: None for relation_name in relations_meta} - - def __contains__(self, key): - return key in self._data - - def __len__(self): - return len(self._data) - - def __iter__(self): - return iter(self._data) - - def __getitem__(self, relation_name): - is_peer = relation_name in self._peers - relation_list = self._data[relation_name] - if relation_list is None: - relation_list = self._data[relation_name] = [] - for rid in self._backend.relation_ids(relation_name): - relation = Relation(relation_name, rid, is_peer, - self._our_unit, self._backend, self._cache) - relation_list.append(relation) - return relation_list - - def _invalidate(self, relation_name): - self._data[relation_name] = None - - def _get_unique(self, relation_name, relation_id=None): - if relation_id is not None: - if not isinstance(relation_id, int): - raise ModelError('relation id {} must be int or None not {}'.format( - relation_id, - type(relation_id).__name__)) - for relation in self[relation_name]: - if relation.id == relation_id: - return relation - else: - # The relation may be dead, but it is not forgotten. - is_peer = relation_name in self._peers - return Relation(relation_name, relation_id, is_peer, - self._our_unit, self._backend, self._cache) - num_related = len(self[relation_name]) - if num_related == 0: - return None - elif num_related == 1: - return self[relation_name][0] - else: - # TODO: We need something in the framework to catch and gracefully handle - # errors, ideally integrating the error catching with Juju's mechanisms. - raise TooManyRelatedAppsError(relation_name, num_related, 1) - - -class BindingMapping: - - def __init__(self, backend): - self._backend = backend - self._data = {} - - def get(self, binding_key): - if isinstance(binding_key, Relation): - binding_name = binding_key.name - relation_id = binding_key.id - elif isinstance(binding_key, str): - binding_name = binding_key - relation_id = None - else: - raise ModelError('binding key must be str or relation instance, not {}' - ''.format(type(binding_key).__name__)) - binding = self._data.get(binding_key) - if binding is None: - binding = Binding(binding_name, relation_id, self._backend) - self._data[binding_key] = binding - return binding - - -class Binding: - """Binding to a network space.""" - - def __init__(self, name, relation_id, backend): - self.name = name - self._relation_id = relation_id - self._backend = backend - self._network = None - - @property - def network(self): - if self._network is None: - try: - self._network = Network(self._backend.network_get(self.name, self._relation_id)) - except RelationNotFoundError: - if self._relation_id is None: - raise - # If a relation is dead, we can still get network info associated with an - # endpoint itself - self._network = Network(self._backend.network_get(self.name)) - return self._network - - -class Network: - """Network space details.""" - - def __init__(self, network_info): - self.interfaces = [] - # Treat multiple addresses on an interface as multiple logical - # interfaces with the same name. - for interface_info in network_info['bind-addresses']: - interface_name = interface_info['interface-name'] - for address_info in interface_info['addresses']: - self.interfaces.append(NetworkInterface(interface_name, address_info)) - self.ingress_addresses = [] - for address in network_info['ingress-addresses']: - self.ingress_addresses.append(ipaddress.ip_address(address)) - self.egress_subnets = [] - for subnet in network_info['egress-subnets']: - self.egress_subnets.append(ipaddress.ip_network(subnet)) - - @property - def bind_address(self): - return self.interfaces[0].address - - @property - def ingress_address(self): - return self.ingress_addresses[0] - - -class NetworkInterface: - - def __init__(self, name, address_info): - self.name = name - # TODO: expose a hardware address here, see LP: #1864070. - self.address = ipaddress.ip_address(address_info['value']) - cidr = address_info['cidr'] - if not cidr: - # The cidr field may be empty, see LP: #1864102. - # In this case, make it a /32 or /128 IP network. - self.subnet = ipaddress.ip_network(address_info['value']) - else: - self.subnet = ipaddress.ip_network(cidr) - # TODO: expose a hostname/canonical name for the address here, see LP: #1864086. - - -class Relation: - def __init__(self, relation_name, relation_id, is_peer, our_unit, backend, cache): - self.name = relation_name - self.id = relation_id - self.app = None - self.units = set() - - # For peer relations, both the remote and the local app are the same. - if is_peer: - self.app = our_unit.app - try: - for unit_name in backend.relation_list(self.id): - unit = cache.get(Unit, unit_name) - self.units.add(unit) - if self.app is None: - self.app = unit.app - except RelationNotFoundError: - # If the relation is dead, just treat it as if it has no remote units. - pass - self.data = RelationData(self, our_unit, backend) - - def __repr__(self): - return '<{}.{} {}:{}>'.format(type(self).__module__, - type(self).__name__, - self.name, - self.id) - - -class RelationData(Mapping): - def __init__(self, relation, our_unit, backend): - self.relation = weakref.proxy(relation) - self._data = { - our_unit: RelationDataContent(self.relation, our_unit, backend), - our_unit.app: RelationDataContent(self.relation, our_unit.app, backend), - } - self._data.update({ - unit: RelationDataContent(self.relation, unit, backend) - for unit in self.relation.units}) - # The relation might be dead so avoid a None key here. - if self.relation.app is not None: - self._data.update({ - self.relation.app: RelationDataContent(self.relation, self.relation.app, backend), - }) - - def __contains__(self, key): - return key in self._data - - def __len__(self): - return len(self._data) - - def __iter__(self): - return iter(self._data) - - def __getitem__(self, key): - return self._data[key] - - -# We mix in MutableMapping here to get some convenience implementations, but whether it's actually -# mutable or not is controlled by the flag. -class RelationDataContent(LazyMapping, MutableMapping): - - def __init__(self, relation, entity, backend): - self.relation = relation - self._entity = entity - self._backend = backend - self._is_app = isinstance(entity, Application) - - def _load(self): - try: - return self._backend.relation_get(self.relation.id, self._entity.name, self._is_app) - except RelationNotFoundError: - # Dead relations tell no tales (and have no data). - return {} - - def _is_mutable(self): - if self._is_app: - is_our_app = self._backend.app_name == self._entity.name - if not is_our_app: - return False - # Whether the application data bag is mutable or not depends on - # whether this unit is a leader or not, but this is not guaranteed - # to be always true during the same hook execution. - return self._backend.is_leader() - else: - is_our_unit = self._backend.unit_name == self._entity.name - if is_our_unit: - return True - return False - - def __setitem__(self, key, value): - if not self._is_mutable(): - raise RelationDataError('cannot set relation data for {}'.format(self._entity.name)) - if not isinstance(value, str): - raise RelationDataError('relation data values must be strings') - - self._backend.relation_set(self.relation.id, key, value, self._is_app) - - # Don't load data unnecessarily if we're only updating. - if self._lazy_data is not None: - if value == '': - # Match the behavior of Juju, which is that setting the value to an - # empty string will remove the key entirely from the relation data. - del self._data[key] - else: - self._data[key] = value - - def __delitem__(self, key): - # Match the behavior of Juju, which is that setting the value to an empty - # string will remove the key entirely from the relation data. - self.__setitem__(key, '') - - -class ConfigData(LazyMapping): - - def __init__(self, backend): - self._backend = backend - - def _load(self): - return self._backend.config_get() - - -class StatusBase: - """Status values specific to applications and units.""" - - _statuses = {} - - def __init__(self, message): - self.message = message - - def __new__(cls, *args, **kwargs): - if cls is StatusBase: - raise TypeError("cannot instantiate a base class") - cls._statuses[cls.name] = cls - return super().__new__(cls) - - @classmethod - def from_name(cls, name, message): - return cls._statuses[name](message) - - -class ActiveStatus(StatusBase): - """The unit is ready. - - The unit believes it is correctly offering all the services it has been asked to offer. - """ - name = 'active' - - def __init__(self, message=None): - super().__init__(message or '') - - -class BlockedStatus(StatusBase): - """The unit requires manual intervention. - - An operator has to manually intervene to unblock the unit and let it proceed. - """ - name = 'blocked' - - -class MaintenanceStatus(StatusBase): - """The unit is performing maintenance tasks. - - The unit is not yet providing services, but is actively doing work in preparation - for providing those services. This is a "spinning" state, not an error state. It - reflects activity on the unit itself, not on peers or related units. - - """ - name = 'maintenance' - - -class UnknownStatus(StatusBase): - """The unit status is unknown. - - A unit-agent has finished calling install, config-changed and start, but the - charm has not called status-set yet. - - """ - name = 'unknown' - - def __init__(self): - # Unknown status cannot be set and does not have a message associated with it. - super().__init__('') - - -class WaitingStatus(StatusBase): - """A unit is unable to progress. - - The unit is unable to progress to an active state because an application to which - it is related is not running. - - """ - name = 'waiting' - - -class Resources: - """Object representing resources for the charm. - """ - - def __init__(self, names, backend): - self._backend = backend - self._paths = {name: None for name in names} - - def fetch(self, name): - """Fetch the resource from the controller or store. - - If successfully fetched, this returns a Path object to where the resource is stored - on disk, otherwise it raises a ModelError. - """ - if name not in self._paths: - raise RuntimeError('invalid resource name: {}'.format(name)) - if self._paths[name] is None: - self._paths[name] = Path(self._backend.resource_get(name)) - return self._paths[name] - - -class Pod: - def __init__(self, backend): - self._backend = backend - - def set_spec(self, spec, k8s_resources=None): - if not self._backend.is_leader(): - raise ModelError('cannot set a pod spec as this unit is not a leader') - self._backend.pod_spec_set(spec, k8s_resources) - - -class StorageMapping(Mapping): - """Map of storage names to lists of Storage instances.""" - - def __init__(self, storage_names, backend): - self._backend = backend - self._storage_map = {storage_name: None for storage_name in storage_names} - - def __contains__(self, key): - return key in self._storage_map - - def __len__(self): - return len(self._storage_map) - - def __iter__(self): - return iter(self._storage_map) - - def __getitem__(self, storage_name): - storage_list = self._storage_map[storage_name] - if storage_list is None: - storage_list = self._storage_map[storage_name] = [] - for storage_id in self._backend.storage_list(storage_name): - storage_list.append(Storage(storage_name, storage_id, self._backend)) - return storage_list - - def request(self, storage_name, count=1): - """Requests new storage instances of a given name. - - Uses storage-add tool to request additional storage. Juju will notify the unit - via -storage-attached events when it becomes available. - """ - if storage_name not in self._storage_map: - raise ModelError(('cannot add storage {!r}:' - ' it is not present in the charm metadata').format(storage_name)) - self._backend.storage_add(storage_name, count) - - -class Storage: - - def __init__(self, storage_name, storage_id, backend): - self.name = storage_name - self.id = storage_id - self._backend = backend - self._location = None - - @property - def location(self): - if self._location is None: - raw = self._backend.storage_get('{}/{}'.format(self.name, self.id), "location") - self._location = Path(raw) - return self._location - - -class ModelError(Exception): - pass - - -class TooManyRelatedAppsError(ModelError): - def __init__(self, relation_name, num_related, max_supported): - super().__init__('Too many remote applications on {} ({} > {})'.format( - relation_name, num_related, max_supported)) - self.relation_name = relation_name - self.num_related = num_related - self.max_supported = max_supported - - -class RelationDataError(ModelError): - pass - - -class RelationNotFoundError(ModelError): - pass - - -class InvalidStatusError(ModelError): - pass - - -class ModelBackend: - - LEASE_RENEWAL_PERIOD = datetime.timedelta(seconds=30) - - def __init__(self): - self.unit_name = os.environ['JUJU_UNIT_NAME'] - self.app_name = self.unit_name.split('/')[0] - - self._is_leader = None - self._leader_check_time = None - - def _run(self, *args, return_output=False, use_json=False): - kwargs = dict(stdout=PIPE, stderr=PIPE) - if use_json: - args += ('--format=json',) - try: - result = run(args, check=True, **kwargs) - except CalledProcessError as e: - raise ModelError(e.stderr) - if return_output: - if result.stdout is None: - return '' - else: - text = result.stdout.decode('utf8') - if use_json: - return json.loads(text) - else: - return text - - def relation_ids(self, relation_name): - relation_ids = self._run('relation-ids', relation_name, return_output=True, use_json=True) - return [int(relation_id.split(':')[-1]) for relation_id in relation_ids] - - def relation_list(self, relation_id): - try: - return self._run('relation-list', '-r', str(relation_id), - return_output=True, use_json=True) - except ModelError as e: - if 'relation not found' in str(e): - raise RelationNotFoundError() from e - raise - - def relation_get(self, relation_id, member_name, is_app): - if not isinstance(is_app, bool): - raise TypeError('is_app parameter to relation_get must be a boolean') - - try: - return self._run('relation-get', '-r', str(relation_id), - '-', member_name, '--app={}'.format(is_app), - return_output=True, use_json=True) - except ModelError as e: - if 'relation not found' in str(e): - raise RelationNotFoundError() from e - raise - - def relation_set(self, relation_id, key, value, is_app): - if not isinstance(is_app, bool): - raise TypeError('is_app parameter to relation_set must be a boolean') - - try: - return self._run('relation-set', '-r', str(relation_id), - '{}={}'.format(key, value), '--app={}'.format(is_app)) - except ModelError as e: - if 'relation not found' in str(e): - raise RelationNotFoundError() from e - raise - - def config_get(self): - return self._run('config-get', return_output=True, use_json=True) - - def is_leader(self): - """Obtain the current leadership status for the unit the charm code is executing on. - - The value is cached for the duration of a lease which is 30s in Juju. - """ - now = time.monotonic() - if self._leader_check_time is None: - check = True - else: - time_since_check = datetime.timedelta(seconds=now - self._leader_check_time) - check = (time_since_check > self.LEASE_RENEWAL_PERIOD or self._is_leader is None) - if check: - # Current time MUST be saved before running is-leader to ensure the cache - # is only used inside the window that is-leader itself asserts. - self._leader_check_time = now - self._is_leader = self._run('is-leader', return_output=True, use_json=True) - - return self._is_leader - - def resource_get(self, resource_name): - return self._run('resource-get', resource_name, return_output=True).strip() - - def pod_spec_set(self, spec, k8s_resources): - tmpdir = Path(tempfile.mkdtemp('-pod-spec-set')) - try: - spec_path = tmpdir / 'spec.json' - spec_path.write_text(json.dumps(spec)) - args = ['--file', str(spec_path)] - if k8s_resources: - k8s_res_path = tmpdir / 'k8s-resources.json' - k8s_res_path.write_text(json.dumps(k8s_resources)) - args.extend(['--k8s-resources', str(k8s_res_path)]) - self._run('pod-spec-set', *args) - finally: - shutil.rmtree(str(tmpdir)) - - def status_get(self, *, is_app=False): - """Get a status of a unit or an application. - - app -- A boolean indicating whether the status should be retrieved for a unit - or an application. - """ - return self._run('status-get', '--include-data', '--application={}'.format(is_app)) - - def status_set(self, status, message='', *, is_app=False): - """Set a status of a unit or an application. - - app -- A boolean indicating whether the status should be set for a unit or an - application. - """ - if not isinstance(is_app, bool): - raise TypeError('is_app parameter must be boolean') - return self._run('status-set', '--application={}'.format(is_app), status, message) - - def storage_list(self, name): - return [int(s.split('/')[1]) for s in self._run('storage-list', name, - return_output=True, use_json=True)] - - def storage_get(self, storage_name_id, attribute): - return self._run('storage-get', '-s', storage_name_id, attribute, - return_output=True, use_json=True) - - def storage_add(self, name, count=1): - if not isinstance(count, int) or isinstance(count, bool): - raise TypeError('storage count must be integer, got: {} ({})'.format(count, - type(count))) - self._run('storage-add', '{}={}'.format(name, count)) - - def action_get(self): - return self._run('action-get', return_output=True, use_json=True) - - def action_set(self, results): - self._run('action-set', *["{}={}".format(k, v) for k, v in results.items()]) - - def action_log(self, message): - self._run('action-log', message) - - def action_fail(self, message=''): - self._run('action-fail', message) - - def application_version_set(self, version): - self._run('application-version-set', '--', version) - - def juju_log(self, level, message): - self._run('juju-log', '--log-level', level, message) - - def network_get(self, binding_name, relation_id=None): - """Return network info provided by network-get for a given binding. - - binding_name -- A name of a binding (relation name or extra-binding name). - relation_id -- An optional relation id to get network info for. - """ - cmd = ['network-get', binding_name] - if relation_id is not None: - cmd.extend(['-r', str(relation_id)]) - try: - return self._run(*cmd, return_output=True, use_json=True) - except ModelError as e: - if 'relation not found' in str(e): - raise RelationNotFoundError() from e - raise - - def add_metrics(self, metrics, labels=None): - cmd = ['add-metric'] - - if labels: - label_args = [] - for k, v in labels.items(): - _ModelBackendValidator.validate_metric_label(k) - _ModelBackendValidator.validate_label_value(k, v) - label_args.append('{}={}'.format(k, v)) - cmd.extend(['--labels', ','.join(label_args)]) - - metric_args = [] - for k, v in metrics.items(): - _ModelBackendValidator.validate_metric_key(k) - metric_value = _ModelBackendValidator.format_metric_value(v) - metric_args.append('{}={}'.format(k, metric_value)) - cmd.extend(metric_args) - self._run(*cmd) - - -class _ModelBackendValidator: - """Provides facilities for validating inputs and formatting them for model backends.""" - - METRIC_KEY_REGEX = re.compile(r'^[a-zA-Z](?:[a-zA-Z0-9-_]*[a-zA-Z0-9])?$') - - @classmethod - def validate_metric_key(cls, key): - if cls.METRIC_KEY_REGEX.match(key) is None: - raise ModelError( - 'invalid metric key {!r}: must match {}'.format( - key, cls.METRIC_KEY_REGEX.pattern)) - - @classmethod - def validate_metric_label(cls, label_name): - if cls.METRIC_KEY_REGEX.match(label_name) is None: - raise ModelError( - 'invalid metric label name {!r}: must match {}'.format( - label_name, cls.METRIC_KEY_REGEX.pattern)) - - @classmethod - def format_metric_value(cls, value): - try: - decimal_value = decimal.Decimal.from_float(value) - except TypeError as e: - e2 = ModelError('invalid metric value {!r} provided:' - ' must be a positive finite float'.format(value)) - raise e2 from e - if decimal_value.is_nan() or decimal_value.is_infinite() or decimal_value < 0: - raise ModelError('invalid metric value {!r} provided:' - ' must be a positive finite float'.format(value)) - return str(decimal_value) - - @classmethod - def validate_label_value(cls, label, value): - # Label values cannot be empty, contain commas or equal signs as those are - # used by add-metric as separators. - if not value: - raise ModelError( - 'metric label {} has an empty value, which is not allowed'.format(label)) - v = str(value) - if re.search('[,=]', v) is not None: - raise ModelError( - 'metric label values must not contain "," or "=": {}={!r}'.format(label, value)) diff --git a/magma/squid_cnf/charms/squid/mod/operator/ops/testing.py b/magma/squid_cnf/charms/squid/mod/operator/ops/testing.py deleted file mode 100644 index 72d840c4b10a2fbbdd53594c0752c63da3da14dd..0000000000000000000000000000000000000000 --- a/magma/squid_cnf/charms/squid/mod/operator/ops/testing.py +++ /dev/null @@ -1,477 +0,0 @@ -# Copyright 2020 Canonical Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import inspect -import pathlib -from textwrap import dedent -import typing - -from ops import charm, framework, model - - -# OptionalYAML is something like metadata.yaml or actions.yaml. You can -# pass in a file-like object or the string directly. -OptionalYAML = typing.Optional[typing.Union[str, typing.TextIO]] - - -# noinspection PyProtectedMember -class Harness: - """This class represents a way to build up the model that will drive a test suite. - - The model that is created is from the viewpoint of the charm that you are testing. - - Example:: - - harness = Harness(MyCharm) - # Do initial setup here - relation_id = harness.add_relation('db', 'postgresql') - # Now instantiate the charm to see events as the model changes - harness.begin() - harness.add_relation_unit(relation_id, 'postgresql/0') - harness.update_relation_data(relation_id, 'postgresql/0', {'key': 'val'}) - # Check that charm has properly handled the relation_joined event for postgresql/0 - self.assertEqual(harness.charm. ...) - - Args: - charm_cls: The Charm class that you'll be testing. - meta: charm.CharmBase is a A string or file-like object containing the contents of - metadata.yaml. If not supplied, we will look for a 'metadata.yaml' file in the - parent directory of the Charm, and if not found fall back to a trivial - 'name: test-charm' metadata. - actions: A string or file-like object containing the contents of - actions.yaml. If not supplied, we will look for a 'actions.yaml' file in the - parent directory of the Charm. - """ - - def __init__( - self, - charm_cls: typing.Type[charm.CharmBase], - *, - meta: OptionalYAML = None, - actions: OptionalYAML = None): - # TODO: jam 2020-03-05 We probably want to take config as a parameter as well, since - # it would define the default values of config that the charm would see. - self._charm_cls = charm_cls - self._charm = None - self._charm_dir = 'no-disk-path' # this may be updated by _create_meta - self._meta = self._create_meta(meta, actions) - self._unit_name = self._meta.name + '/0' - self._framework = None - self._hooks_enabled = True - self._relation_id_counter = 0 - self._backend = _TestingModelBackend(self._unit_name, self._meta) - self._model = model.Model(self._unit_name, self._meta, self._backend) - self._framework = framework.Framework(":memory:", self._charm_dir, self._meta, self._model) - - @property - def charm(self) -> charm.CharmBase: - """Return the instance of the charm class that was passed to __init__. - - Note that the Charm is not instantiated until you have called - :meth:`.begin()`. - """ - return self._charm - - @property - def model(self) -> model.Model: - """Return the :class:`~ops.model.Model` that is being driven by this Harness.""" - return self._model - - @property - def framework(self) -> framework.Framework: - """Return the Framework that is being driven by this Harness.""" - return self._framework - - def begin(self) -> None: - """Instantiate the Charm and start handling events. - - Before calling begin(), there is no Charm instance, so changes to the Model won't emit - events. You must call begin before :attr:`.charm` is valid. - """ - if self._charm is not None: - raise RuntimeError('cannot call the begin method on the harness more than once') - - # The Framework adds attributes to class objects for events, etc. As such, we can't re-use - # the original class against multiple Frameworks. So create a locally defined class - # and register it. - # TODO: jam 2020-03-16 We are looking to changes this to Instance attributes instead of - # Class attributes which should clean up this ugliness. The API can stay the same - class TestEvents(self._charm_cls.on.__class__): - pass - - TestEvents.__name__ = self._charm_cls.on.__class__.__name__ - - class TestCharm(self._charm_cls): - on = TestEvents() - - # Note: jam 2020-03-01 This is so that errors in testing say MyCharm has no attribute foo, - # rather than TestCharm has no attribute foo. - TestCharm.__name__ = self._charm_cls.__name__ - self._charm = TestCharm(self._framework, self._framework.meta.name) - - def _create_meta(self, charm_metadata, action_metadata): - """Create a CharmMeta object. - - Handle the cases where a user doesn't supply explicit metadata snippets. - """ - filename = inspect.getfile(self._charm_cls) - charm_dir = pathlib.Path(filename).parents[1] - - if charm_metadata is None: - metadata_path = charm_dir / 'metadata.yaml' - if metadata_path.is_file(): - charm_metadata = metadata_path.read_text() - self._charm_dir = charm_dir - else: - # The simplest of metadata that the framework can support - charm_metadata = 'name: test-charm' - elif isinstance(charm_metadata, str): - charm_metadata = dedent(charm_metadata) - - if action_metadata is None: - actions_path = charm_dir / 'actions.yaml' - if actions_path.is_file(): - action_metadata = actions_path.read_text() - self._charm_dir = charm_dir - elif isinstance(action_metadata, str): - action_metadata = dedent(action_metadata) - - return charm.CharmMeta.from_yaml(charm_metadata, action_metadata) - - def disable_hooks(self) -> None: - """Stop emitting hook events when the model changes. - - This can be used by developers to stop changes to the model from emitting events that - the charm will react to. Call :meth:`.enable_hooks` - to re-enable them. - """ - self._hooks_enabled = False - - def enable_hooks(self) -> None: - """Re-enable hook events from charm.on when the model is changed. - - By default hook events are enabled once you call :meth:`.begin`, - but if you have used :meth:`.disable_hooks`, this can be used to - enable them again. - """ - self._hooks_enabled = True - - def _next_relation_id(self): - rel_id = self._relation_id_counter - self._relation_id_counter += 1 - return rel_id - - def add_relation(self, relation_name: str, remote_app: str) -> int: - """Declare that there is a new relation between this app and `remote_app`. - - Args: - relation_name: The relation on Charm that is being related to - remote_app: The name of the application that is being related to - - Return: - The relation_id created by this add_relation. - """ - rel_id = self._next_relation_id() - self._backend._relation_ids_map.setdefault(relation_name, []).append(rel_id) - self._backend._relation_names[rel_id] = relation_name - self._backend._relation_list_map[rel_id] = [] - self._backend._relation_data[rel_id] = { - remote_app: {}, - self._backend.unit_name: {}, - self._backend.app_name: {}, - } - # Reload the relation_ids list - if self._model is not None: - self._model.relations._invalidate(relation_name) - if self._charm is None or not self._hooks_enabled: - return rel_id - relation = self._model.get_relation(relation_name, rel_id) - app = self._model.get_app(remote_app) - self._charm.on[relation_name].relation_created.emit( - relation, app) - return rel_id - - def add_relation_unit(self, relation_id: int, remote_unit_name: str) -> None: - """Add a new unit to a relation. - - Example:: - - rel_id = harness.add_relation('db', 'postgresql') - harness.add_relation_unit(rel_id, 'postgresql/0') - - This will trigger a `relation_joined` event and a `relation_changed` event. - - Args: - relation_id: The integer relation identifier (as returned by add_relation). - remote_unit_name: A string representing the remote unit that is being added. - Return: - None - """ - self._backend._relation_list_map[relation_id].append(remote_unit_name) - self._backend._relation_data[relation_id][remote_unit_name] = {} - relation_name = self._backend._relation_names[relation_id] - # Make sure that the Model reloads the relation_list for this relation_id, as well as - # reloading the relation data for this unit. - if self._model is not None: - self._model.relations._invalidate(relation_name) - remote_unit = self._model.get_unit(remote_unit_name) - relation = self._model.get_relation(relation_name, relation_id) - relation.data[remote_unit]._invalidate() - if self._charm is None or not self._hooks_enabled: - return - self._charm.on[relation_name].relation_joined.emit( - relation, remote_unit.app, remote_unit) - - def get_relation_data(self, relation_id: int, app_or_unit: str) -> typing.Mapping: - """Get the relation data bucket for a single app or unit in a given relation. - - This ignores all of the safety checks of who can and can't see data in relations (eg, - non-leaders can't read their own application's relation data because there are no events - that keep that data up-to-date for the unit). - - Args: - relation_id: The relation whose content we want to look at. - app_or_unit: The name of the application or unit whose data we want to read - Return: - a dict containing the relation data for `app_or_unit` or None. - Raises: - KeyError: if relation_id doesn't exist - """ - return self._backend._relation_data[relation_id].get(app_or_unit, None) - - def get_workload_version(self) -> str: - """Read the workload version that was set by the unit.""" - return self._backend._workload_version - - def update_relation_data( - self, - relation_id: int, - app_or_unit: str, - key_values: typing.Mapping, - ) -> None: - """Update the relation data for a given unit or application in a given relation. - - This also triggers the `relation_changed` event for this relation_id. - - Args: - relation_id: The integer relation_id representing this relation. - app_or_unit: The unit or application name that is being updated. - This can be the local or remote application. - key_values: Each key/value will be updated in the relation data. - """ - relation_name = self._backend._relation_names[relation_id] - relation = self._model.get_relation(relation_name, relation_id) - if '/' in app_or_unit: - entity = self._model.get_unit(app_or_unit) - else: - entity = self._model.get_app(app_or_unit) - rel_data = relation.data.get(entity, None) - if rel_data is not None: - # rel_data may have cached now-stale data, so _invalidate() it. - # Note, this won't cause the data to be loaded if it wasn't already. - rel_data._invalidate() - - new_values = self._backend._relation_data[relation_id][app_or_unit].copy() - for k, v in key_values.items(): - if v == '': - new_values.pop(k, None) - else: - new_values[k] = v - self._backend._relation_data[relation_id][app_or_unit] = new_values - - if app_or_unit == self._model.unit.name: - # No events for our own unit - return - if app_or_unit == self._model.app.name: - # updating our own app only generates an event if it is a peer relation and we - # aren't the leader - is_peer = self._meta.relations[relation_name].role == 'peers' - if not is_peer: - return - if self._model.unit.is_leader(): - return - self._emit_relation_changed(relation_id, app_or_unit) - - def _emit_relation_changed(self, relation_id, app_or_unit): - if self._charm is None or not self._hooks_enabled: - return - rel_name = self._backend._relation_names[relation_id] - relation = self.model.get_relation(rel_name, relation_id) - if '/' in app_or_unit: - app_name = app_or_unit.split('/')[0] - unit_name = app_or_unit - app = self.model.get_app(app_name) - unit = self.model.get_unit(unit_name) - args = (relation, app, unit) - else: - app_name = app_or_unit - app = self.model.get_app(app_name) - args = (relation, app) - self._charm.on[rel_name].relation_changed.emit(*args) - - def update_config( - self, - key_values: typing.Mapping[str, str] = None, - unset: typing.Iterable[str] = (), - ) -> None: - """Update the config as seen by the charm. - - This will trigger a `config_changed` event. - - Args: - key_values: A Mapping of key:value pairs to update in config. - unset: An iterable of keys to remove from Config. (Note that this does - not currently reset the config values to the default defined in config.yaml.) - """ - config = self._backend._config - if key_values is not None: - for key, value in key_values.items(): - config[key] = value - for key in unset: - config.pop(key, None) - # NOTE: jam 2020-03-01 Note that this sort of works "by accident". Config - # is a LazyMapping, but its _load returns a dict and this method mutates - # the dict that Config is caching. Arguably we should be doing some sort - # of charm.framework.model.config._invalidate() - if self._charm is None or not self._hooks_enabled: - return - self._charm.on.config_changed.emit() - - def set_leader(self, is_leader: bool = True) -> None: - """Set whether this unit is the leader or not. - - If this charm becomes a leader then `leader_elected` will be triggered. - - Args: - is_leader: True/False as to whether this unit is the leader. - """ - was_leader = self._backend._is_leader - self._backend._is_leader = is_leader - # Note: jam 2020-03-01 currently is_leader is cached at the ModelBackend level, not in - # the Model objects, so this automatically gets noticed. - if is_leader and not was_leader and self._charm is not None and self._hooks_enabled: - self._charm.on.leader_elected.emit() - - -class _TestingModelBackend: - """This conforms to the interface for ModelBackend but provides canned data. - - DO NOT use this class directly, it is used by `Harness`_ to drive the model. - `Harness`_ is responsible for maintaining the internal consistency of the values here, - as the only public methods of this type are for implementing ModelBackend. - """ - - def __init__(self, unit_name, meta): - self.unit_name = unit_name - self.app_name = self.unit_name.split('/')[0] - self._calls = [] - self._meta = meta - self._is_leader = None - self._relation_ids_map = {} # relation name to [relation_ids,...] - self._relation_names = {} # reverse map from relation_id to relation_name - self._relation_list_map = {} # relation_id: [unit_name,...] - self._relation_data = {} # {relation_id: {name: data}} - self._config = {} - self._is_leader = False - self._resources_map = {} - self._pod_spec = None - self._app_status = None - self._unit_status = None - self._workload_version = None - - def relation_ids(self, relation_name): - try: - return self._relation_ids_map[relation_name] - except KeyError as e: - if relation_name not in self._meta.relations: - raise model.ModelError('{} is not a known relation'.format(relation_name)) from e - return [] - - def relation_list(self, relation_id): - try: - return self._relation_list_map[relation_id] - except KeyError as e: - raise model.RelationNotFoundError from e - - def relation_get(self, relation_id, member_name, is_app): - if is_app and '/' in member_name: - member_name = member_name.split('/')[0] - if relation_id not in self._relation_data: - raise model.RelationNotFoundError() - return self._relation_data[relation_id][member_name].copy() - - def relation_set(self, relation_id, key, value, is_app): - relation = self._relation_data[relation_id] - if is_app: - bucket_key = self.app_name - else: - bucket_key = self.unit_name - if bucket_key not in relation: - relation[bucket_key] = {} - bucket = relation[bucket_key] - if value == '': - bucket.pop(key, None) - else: - bucket[key] = value - - def config_get(self): - return self._config - - def is_leader(self): - return self._is_leader - - def application_version_set(self, version): - self._workload_version = version - - def resource_get(self, resource_name): - return self._resources_map[resource_name] - - def pod_spec_set(self, spec, k8s_resources): - self._pod_spec = (spec, k8s_resources) - - def status_get(self, *, is_app=False): - if is_app: - return self._app_status - else: - return self._unit_status - - def status_set(self, status, message='', *, is_app=False): - if is_app: - self._app_status = (status, message) - else: - self._unit_status = (status, message) - - def storage_list(self, name): - raise NotImplementedError(self.storage_list) - - def storage_get(self, storage_name_id, attribute): - raise NotImplementedError(self.storage_get) - - def storage_add(self, name, count=1): - raise NotImplementedError(self.storage_add) - - def action_get(self): - raise NotImplementedError(self.action_get) - - def action_set(self, results): - raise NotImplementedError(self.action_set) - - def action_log(self, message): - raise NotImplementedError(self.action_log) - - def action_fail(self, message=''): - raise NotImplementedError(self.action_fail) - - def network_get(self, endpoint_name, relation_id=None): - raise NotImplementedError(self.network_get) diff --git a/magma/squid_cnf/charms/squid/mod/operator/requirements.txt b/magma/squid_cnf/charms/squid/mod/operator/requirements.txt deleted file mode 100644 index 5500f007d0bf6c6098afc0f2c6d00915e345a569..0000000000000000000000000000000000000000 --- a/magma/squid_cnf/charms/squid/mod/operator/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -PyYAML diff --git a/magma/squid_cnf/charms/squid/mod/operator/run_tests b/magma/squid_cnf/charms/squid/mod/operator/run_tests deleted file mode 100755 index 56411030fdcd8629ffbfbbebb8a8a0650203a934..0000000000000000000000000000000000000000 --- a/magma/squid_cnf/charms/squid/mod/operator/run_tests +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash - -python3 -m unittest "$@" diff --git a/magma/squid_cnf/charms/squid/mod/operator/setup.py b/magma/squid_cnf/charms/squid/mod/operator/setup.py deleted file mode 100644 index cc017478d4c6922e4aaf8dc7715152e144cfab7e..0000000000000000000000000000000000000000 --- a/magma/squid_cnf/charms/squid/mod/operator/setup.py +++ /dev/null @@ -1,39 +0,0 @@ -# Copyright 2019 Canonical Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from setuptools import setup - -with open("README.md", "r") as fh: - long_description = fh.read() - -setup( - name="ops", - version="0.0.1", - description="The Python library behind great charms", - long_description=long_description, - long_description_content_type="text/markdown", - license="Apache-2.0", - url="https://github.com/canonical/operator", - packages=["ops"], - classifiers=[ - "Development Status :: 4 - Beta", - - "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.6", - "Programming Language :: Python :: 3.7", - "Programming Language :: Python :: 3.8", - - "License :: OSI Approved :: Apache Software License", - ], -) diff --git a/magma/squid_cnf/charms/squid/mod/operator/test/__init__.py b/magma/squid_cnf/charms/squid/mod/operator/test/__init__.py deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/magma/squid_cnf/charms/squid/mod/operator/test/bin/relation-ids b/magma/squid_cnf/charms/squid/mod/operator/test/bin/relation-ids deleted file mode 100755 index a7e0ead2d3182713bd826696fc403b5a8c54faa6..0000000000000000000000000000000000000000 --- a/magma/squid_cnf/charms/squid/mod/operator/test/bin/relation-ids +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash - -case $1 in - db) echo '["db:1"]' ;; - mon) echo '["mon:2"]' ;; - ha) echo '[]' ;; - db0) echo '[]' ;; - db1) echo '["db1:4"]' ;; - db2) echo '["db2:5", "db2:6"]' ;; - *) echo '[]' ;; -esac diff --git a/magma/squid_cnf/charms/squid/mod/operator/test/bin/relation-list b/magma/squid_cnf/charms/squid/mod/operator/test/bin/relation-list deleted file mode 100755 index 88490159775624108766a17a35a77599ddea8f03..0000000000000000000000000000000000000000 --- a/magma/squid_cnf/charms/squid/mod/operator/test/bin/relation-list +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/bash - -fail_not_found() { - 1>&2 echo "ERROR invalid value \"$1\" for option -r: relation not found" - exit 2 -} - -case $2 in - 1) echo '["remote/0"]' ;; - 2) echo '["remote/0"]' ;; - 3) fail_not_found $2 ;; - 4) echo '["remoteapp1/0"]' ;; - 5) echo '["remoteapp1/0"]' ;; - 6) echo '["remoteapp2/0"]' ;; - *) fail_not_found $2 ;; -esac diff --git a/magma/squid_cnf/charms/squid/mod/operator/test/charms/test_main/config.yaml b/magma/squid_cnf/charms/squid/mod/operator/test/charms/test_main/config.yaml deleted file mode 100644 index ffc0186002391ca52273d39bebcc9c4261c47535..0000000000000000000000000000000000000000 --- a/magma/squid_cnf/charms/squid/mod/operator/test/charms/test_main/config.yaml +++ /dev/null @@ -1 +0,0 @@ -"options": {} diff --git a/magma/squid_cnf/charms/squid/mod/operator/test/charms/test_main/lib/__init__.py b/magma/squid_cnf/charms/squid/mod/operator/test/charms/test_main/lib/__init__.py deleted file mode 100755 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/magma/squid_cnf/charms/squid/mod/operator/test/charms/test_main/lib/ops b/magma/squid_cnf/charms/squid/mod/operator/test/charms/test_main/lib/ops deleted file mode 120000 index 1356790e5ca930db72fe58ee452914193ba47f20..0000000000000000000000000000000000000000 --- a/magma/squid_cnf/charms/squid/mod/operator/test/charms/test_main/lib/ops +++ /dev/null @@ -1 +0,0 @@ -../../../../ops \ No newline at end of file diff --git a/magma/squid_cnf/charms/squid/mod/operator/test/charms/test_main/metadata.yaml b/magma/squid_cnf/charms/squid/mod/operator/test/charms/test_main/metadata.yaml deleted file mode 100644 index 3b3aed87e96121224c63916b04009daf40fcab35..0000000000000000000000000000000000000000 --- a/magma/squid_cnf/charms/squid/mod/operator/test/charms/test_main/metadata.yaml +++ /dev/null @@ -1,26 +0,0 @@ -name: main -summary: A charm used for testing the basic operation of the entrypoint code. -maintainer: Dmitrii Shcherbakov -description: A charm used for testing the basic operation of the entrypoint code. -tags: - - misc -series: - - bionic - - cosmic - - disco -min-juju-version: 2.7.1 -provides: - db: - interface: db -requires: - mon: - interface: monitoring -peers: - ha: - interface: cluster -subordinate: false -storage: - disks: - type: block - multiple: - range: 0- diff --git a/magma/squid_cnf/charms/squid/mod/operator/test/charms/test_main/src/charm.py b/magma/squid_cnf/charms/squid/mod/operator/test/charms/test_main/src/charm.py deleted file mode 100755 index 0e066c97bc7223e2731fb51fb0beb62e3d94de3e..0000000000000000000000000000000000000000 --- a/magma/squid_cnf/charms/squid/mod/operator/test/charms/test_main/src/charm.py +++ /dev/null @@ -1,208 +0,0 @@ -#!/usr/bin/env python3 -# Copyright 2019 Canonical Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -import base64 -import pickle -import sys -import logging - -sys.path.append('lib') - -from ops.charm import CharmBase # noqa: E402 (module-level import after non-import code) -from ops.main import main # noqa: E402 (ditto) - -logger = logging.getLogger() - - -class Charm(CharmBase): - - def __init__(self, *args): - super().__init__(*args) - - # This environment variable controls the test charm behavior. - charm_config = os.environ.get('CHARM_CONFIG') - if charm_config is not None: - self._charm_config = pickle.loads(base64.b64decode(charm_config)) - else: - self._charm_config = {} - - # TODO: refactor to use StoredState - # (this implies refactoring most of test_main.py) - self._state_file = self._charm_config.get('STATE_FILE') - try: - with open(str(self._state_file), 'rb') as f: - self._state = pickle.load(f) - except (FileNotFoundError, EOFError): - self._state = { - 'on_install': [], - 'on_start': [], - 'on_config_changed': [], - 'on_update_status': [], - 'on_leader_settings_changed': [], - 'on_db_relation_joined': [], - 'on_mon_relation_changed': [], - 'on_mon_relation_departed': [], - 'on_ha_relation_broken': [], - 'on_foo_bar_action': [], - 'on_start_action': [], - 'on_collect_metrics': [], - - 'on_log_critical_action': [], - 'on_log_error_action': [], - 'on_log_warning_action': [], - 'on_log_info_action': [], - 'on_log_debug_action': [], - - # Observed event types per invocation. A list is used to preserve the - # order in which charm handlers have observed the events. - 'observed_event_types': [], - } - - self.framework.observe(self.on.install, self) - self.framework.observe(self.on.start, self) - self.framework.observe(self.on.config_changed, self) - self.framework.observe(self.on.update_status, self) - self.framework.observe(self.on.leader_settings_changed, self) - # Test relation events with endpoints from different - # sections (provides, requires, peers) as well. - self.framework.observe(self.on.db_relation_joined, self) - self.framework.observe(self.on.mon_relation_changed, self) - self.framework.observe(self.on.mon_relation_departed, self) - self.framework.observe(self.on.ha_relation_broken, self) - - if self._charm_config.get('USE_ACTIONS'): - self.framework.observe(self.on.start_action, self) - self.framework.observe(self.on.foo_bar_action, self) - - self.framework.observe(self.on.collect_metrics, self) - - if self._charm_config.get('USE_LOG_ACTIONS'): - self.framework.observe(self.on.log_critical_action, self) - self.framework.observe(self.on.log_error_action, self) - self.framework.observe(self.on.log_warning_action, self) - self.framework.observe(self.on.log_info_action, self) - self.framework.observe(self.on.log_debug_action, self) - - def _write_state(self): - """Write state variables so that the parent process can read them. - - Each invocation will override the previous state which is intentional. - """ - if self._state_file is not None: - with self._state_file.open('wb') as f: - pickle.dump(self._state, f) - - def on_install(self, event): - self._state['on_install'].append(type(event)) - self._state['observed_event_types'].append(type(event)) - self._write_state() - - def on_start(self, event): - self._state['on_start'].append(type(event)) - self._state['observed_event_types'].append(type(event)) - self._write_state() - - def on_config_changed(self, event): - self._state['on_config_changed'].append(type(event)) - self._state['observed_event_types'].append(type(event)) - event.defer() - self._write_state() - - def on_update_status(self, event): - self._state['on_update_status'].append(type(event)) - self._state['observed_event_types'].append(type(event)) - self._write_state() - - def on_leader_settings_changed(self, event): - self._state['on_leader_settings_changed'].append(type(event)) - self._state['observed_event_types'].append(type(event)) - self._write_state() - - def on_db_relation_joined(self, event): - assert event.app is not None, 'application name cannot be None for a relation-joined event' - self._state['on_db_relation_joined'].append(type(event)) - self._state['observed_event_types'].append(type(event)) - self._state['db_relation_joined_data'] = event.snapshot() - self._write_state() - - def on_mon_relation_changed(self, event): - assert event.app is not None, ( - 'application name cannot be None for a relation-changed event') - if os.environ.get('JUJU_REMOTE_UNIT'): - assert event.unit is not None, ( - 'a unit name cannot be None for a relation-changed event' - ' associated with a remote unit') - self._state['on_mon_relation_changed'].append(type(event)) - self._state['observed_event_types'].append(type(event)) - self._state['mon_relation_changed_data'] = event.snapshot() - self._write_state() - - def on_mon_relation_departed(self, event): - assert event.app is not None, ( - 'application name cannot be None for a relation-departed event') - self._state['on_mon_relation_departed'].append(type(event)) - self._state['observed_event_types'].append(type(event)) - self._state['mon_relation_departed_data'] = event.snapshot() - self._write_state() - - def on_ha_relation_broken(self, event): - assert event.app is None, ( - 'relation-broken events cannot have a reference to a remote application') - assert event.unit is None, ( - 'relation broken events cannot have a reference to a remote unit') - self._state['on_ha_relation_broken'].append(type(event)) - self._state['observed_event_types'].append(type(event)) - self._state['ha_relation_broken_data'] = event.snapshot() - self._write_state() - - def on_start_action(self, event): - assert event.handle.kind == 'start_action', ( - 'event action name cannot be different from the one being handled') - self._state['on_start_action'].append(type(event)) - self._state['observed_event_types'].append(type(event)) - self._write_state() - - def on_foo_bar_action(self, event): - assert event.handle.kind == 'foo_bar_action', ( - 'event action name cannot be different from the one being handled') - self._state['on_foo_bar_action'].append(type(event)) - self._state['observed_event_types'].append(type(event)) - self._write_state() - - def on_collect_metrics(self, event): - self._state['on_collect_metrics'].append(type(event)) - self._state['observed_event_types'].append(type(event)) - event.add_metrics({'foo': 42}, {'bar': 4.2}) - self._write_state() - - def on_log_critical_action(self, event): - logger.critical('super critical') - - def on_log_error_action(self, event): - logger.error('grave error') - - def on_log_warning_action(self, event): - logger.warning('wise warning') - - def on_log_info_action(self, event): - logger.info('useful info') - - def on_log_debug_action(self, event): - logger.debug('insightful debug') - - -if __name__ == '__main__': - main(Charm) diff --git a/magma/squid_cnf/charms/squid/mod/operator/test/test_charm.py b/magma/squid_cnf/charms/squid/mod/operator/test/test_charm.py deleted file mode 100755 index c8d84475cbc7ec832db108bdfcebea2d16c457b3..0000000000000000000000000000000000000000 --- a/magma/squid_cnf/charms/squid/mod/operator/test/test_charm.py +++ /dev/null @@ -1,323 +0,0 @@ -#!/usr/bin/python3 -# Copyright 2019 Canonical Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -import unittest -import tempfile -import shutil - -from pathlib import Path - -from ops.charm import ( - CharmBase, - CharmMeta, - CharmEvents, -) -from ops.framework import Framework, EventSource, EventBase -from ops.model import Model, ModelBackend - -from .test_helpers import fake_script, fake_script_calls - - -class TestCharm(unittest.TestCase): - - def setUp(self): - def restore_env(env): - os.environ.clear() - os.environ.update(env) - self.addCleanup(restore_env, os.environ.copy()) - - os.environ['PATH'] = "{}:{}".format(Path(__file__).parent / 'bin', os.environ['PATH']) - os.environ['JUJU_UNIT_NAME'] = 'local/0' - - self.tmpdir = Path(tempfile.mkdtemp()) - self.addCleanup(shutil.rmtree, str(self.tmpdir)) - self.meta = CharmMeta() - - class CustomEvent(EventBase): - pass - - class TestCharmEvents(CharmEvents): - custom = EventSource(CustomEvent) - - # Relations events are defined dynamically and modify the class attributes. - # We use a subclass temporarily to prevent these side effects from leaking. - CharmBase.on = TestCharmEvents() - - def cleanup(): - CharmBase.on = CharmEvents() - self.addCleanup(cleanup) - - def create_framework(self): - model = Model('local/0', self.meta, ModelBackend()) - framework = Framework(self.tmpdir / "framework.data", self.tmpdir, self.meta, model) - self.addCleanup(framework.close) - return framework - - def test_basic(self): - - class MyCharm(CharmBase): - - def __init__(self, *args): - super().__init__(*args) - - self.started = False - framework.observe(self.on.start, self) - - def on_start(self, event): - self.started = True - - events = list(MyCharm.on.events()) - self.assertIn('install', events) - self.assertIn('custom', events) - - framework = self.create_framework() - charm = MyCharm(framework, None) - charm.on.start.emit() - - self.assertEqual(charm.started, True) - - def test_helper_properties(self): - framework = self.create_framework() - - class MyCharm(CharmBase): - pass - - charm = MyCharm(framework, None) - self.assertEqual(charm.app, framework.model.app) - self.assertEqual(charm.unit, framework.model.unit) - self.assertEqual(charm.meta, framework.meta) - self.assertEqual(charm.charm_dir, framework.charm_dir) - - def test_relation_events(self): - - class MyCharm(CharmBase): - def __init__(self, *args): - super().__init__(*args) - self.seen = [] - for rel in ('req1', 'req-2', 'pro1', 'pro-2', 'peer1', 'peer-2'): - # Hook up relation events to generic handler. - self.framework.observe(self.on[rel].relation_joined, self.on_any_relation) - self.framework.observe(self.on[rel].relation_changed, self.on_any_relation) - self.framework.observe(self.on[rel].relation_departed, self.on_any_relation) - self.framework.observe(self.on[rel].relation_broken, self.on_any_relation) - - def on_any_relation(self, event): - assert event.relation.name == 'req1' - assert event.relation.app.name == 'remote' - self.seen.append(type(event).__name__) - - # language=YAML - self.meta = CharmMeta.from_yaml(metadata=''' -name: my-charm -requires: - req1: - interface: req1 - req-2: - interface: req2 -provides: - pro1: - interface: pro1 - pro-2: - interface: pro2 -peers: - peer1: - interface: peer1 - peer-2: - interface: peer2 -''') - - charm = MyCharm(self.create_framework(), None) - - rel = charm.framework.model.get_relation('req1', 1) - unit = charm.framework.model.get_unit('remote/0') - charm.on['req1'].relation_joined.emit(rel, unit) - charm.on['req1'].relation_changed.emit(rel, unit) - charm.on['req-2'].relation_changed.emit(rel, unit) - charm.on['pro1'].relation_departed.emit(rel, unit) - charm.on['pro-2'].relation_departed.emit(rel, unit) - charm.on['peer1'].relation_broken.emit(rel) - charm.on['peer-2'].relation_broken.emit(rel) - - self.assertEqual(charm.seen, [ - 'RelationJoinedEvent', - 'RelationChangedEvent', - 'RelationChangedEvent', - 'RelationDepartedEvent', - 'RelationDepartedEvent', - 'RelationBrokenEvent', - 'RelationBrokenEvent', - ]) - - def test_storage_events(self): - - class MyCharm(CharmBase): - def __init__(self, *args): - super().__init__(*args) - self.seen = [] - self.framework.observe(self.on['stor1'].storage_attached, self) - self.framework.observe(self.on['stor2'].storage_detaching, self) - self.framework.observe(self.on['stor3'].storage_attached, self) - self.framework.observe(self.on['stor-4'].storage_attached, self) - - def on_stor1_storage_attached(self, event): - self.seen.append(type(event).__name__) - - def on_stor2_storage_detaching(self, event): - self.seen.append(type(event).__name__) - - def on_stor3_storage_attached(self, event): - self.seen.append(type(event).__name__) - - def on_stor_4_storage_attached(self, event): - self.seen.append(type(event).__name__) - - # language=YAML - self.meta = CharmMeta.from_yaml(''' -name: my-charm -storage: - stor-4: - multiple: - range: 2-4 - type: filesystem - stor1: - type: filesystem - stor2: - multiple: - range: "2" - type: filesystem - stor3: - multiple: - range: 2- - type: filesystem -''') - - self.assertIsNone(self.meta.storages['stor1'].multiple_range) - self.assertEqual(self.meta.storages['stor2'].multiple_range, (2, 2)) - self.assertEqual(self.meta.storages['stor3'].multiple_range, (2, None)) - self.assertEqual(self.meta.storages['stor-4'].multiple_range, (2, 4)) - - charm = MyCharm(self.create_framework(), None) - - charm.on['stor1'].storage_attached.emit() - charm.on['stor2'].storage_detaching.emit() - charm.on['stor3'].storage_attached.emit() - charm.on['stor-4'].storage_attached.emit() - - self.assertEqual(charm.seen, [ - 'StorageAttachedEvent', - 'StorageDetachingEvent', - 'StorageAttachedEvent', - 'StorageAttachedEvent', - ]) - - @classmethod - def _get_action_test_meta(cls): - # language=YAML - return CharmMeta.from_yaml(metadata=''' -name: my-charm -''', actions=''' -foo-bar: - description: "Foos the bar." - params: - foo-name: - description: "A foo name to bar" - type: string - silent: - default: false - description: "" - type: boolean - required: foo-bar - title: foo-bar -start: - description: "Start the unit." -''') - - def _test_action_events(self, cmd_type): - - class MyCharm(CharmBase): - - def __init__(self, *args): - super().__init__(*args) - framework.observe(self.on.foo_bar_action, self) - framework.observe(self.on.start_action, self) - - def on_foo_bar_action(self, event): - self.seen_action_params = event.params - event.log('test-log') - event.set_results({'res': 'val with spaces'}) - event.fail('test-fail') - - def on_start_action(self, event): - pass - - fake_script(self, cmd_type + '-get', """echo '{"foo-name": "name", "silent": true}'""") - fake_script(self, cmd_type + '-set', "") - fake_script(self, cmd_type + '-log', "") - fake_script(self, cmd_type + '-fail', "") - self.meta = self._get_action_test_meta() - - os.environ['JUJU_{}_NAME'.format(cmd_type.upper())] = 'foo-bar' - framework = self.create_framework() - charm = MyCharm(framework, None) - - events = list(MyCharm.on.events()) - self.assertIn('foo_bar_action', events) - self.assertIn('start_action', events) - - charm.on.foo_bar_action.emit() - self.assertEqual(charm.seen_action_params, {"foo-name": "name", "silent": True}) - self.assertEqual(fake_script_calls(self), [ - [cmd_type + '-get', '--format=json'], - [cmd_type + '-log', "test-log"], - [cmd_type + '-set', "res=val with spaces"], - [cmd_type + '-fail', "test-fail"], - ]) - - # Make sure that action events that do not match the current context are - # not possible to emit by hand. - with self.assertRaises(RuntimeError): - charm.on.start_action.emit() - - def test_action_events(self): - self._test_action_events('action') - - def _test_action_event_defer_fails(self, cmd_type): - - class MyCharm(CharmBase): - - def __init__(self, *args): - super().__init__(*args) - framework.observe(self.on.start_action, self) - - def on_start_action(self, event): - event.defer() - - fake_script(self, cmd_type + '-get', """echo '{"foo-name": "name", "silent": true}'""") - self.meta = self._get_action_test_meta() - - os.environ['JUJU_{}_NAME'.format(cmd_type.upper())] = 'start' - framework = self.create_framework() - charm = MyCharm(framework, None) - - with self.assertRaises(RuntimeError): - charm.on.start_action.emit() - - def test_action_event_defer_fails(self): - self._test_action_event_defer_fails('action') - - -if __name__ == "__main__": - unittest.main() diff --git a/magma/squid_cnf/charms/squid/mod/operator/test/test_framework.py b/magma/squid_cnf/charms/squid/mod/operator/test/test_framework.py deleted file mode 100644 index 2ec4c4ed56d9b18aef419107a97054c557cc4218..0000000000000000000000000000000000000000 --- a/magma/squid_cnf/charms/squid/mod/operator/test/test_framework.py +++ /dev/null @@ -1,1727 +0,0 @@ -# Copyright 2019-2020 Canonical Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import datetime -import gc -import inspect -import io -import os -import shutil -import sys -import tempfile -import unittest -from unittest.mock import patch -from pathlib import Path - -from ops import charm, model -from ops.framework import ( - _BREAKPOINT_WELCOME_MESSAGE, - BoundStoredState, - CommitEvent, - EventBase, - ObjectEvents, - EventSource, - Framework, - Handle, - NoSnapshotError, - Object, - PreCommitEvent, - SQLiteStorage, - StoredList, - StoredState, - StoredStateData, -) -from test.test_helpers import fake_script - - -class TestFramework(unittest.TestCase): - - def setUp(self): - self.tmpdir = Path(tempfile.mkdtemp()) - self.addCleanup(shutil.rmtree, str(self.tmpdir)) - default_timeout = SQLiteStorage.DB_LOCK_TIMEOUT - - def timeout_cleanup(): - SQLiteStorage.DB_LOCK_TIMEOUT = default_timeout - SQLiteStorage.DB_LOCK_TIMEOUT = datetime.timedelta(0) - self.addCleanup(timeout_cleanup) - - def create_framework(self): - framework = Framework(self.tmpdir / "framework.data", self.tmpdir, None, None) - self.addCleanup(framework.close) - return framework - - def test_handle_path(self): - cases = [ - (Handle(None, "root", None), "root"), - (Handle(None, "root", "1"), "root[1]"), - (Handle(Handle(None, "root", None), "child", None), "root/child"), - (Handle(Handle(None, "root", "1"), "child", "2"), "root[1]/child[2]"), - ] - for handle, path in cases: - self.assertEqual(str(handle), path) - self.assertEqual(Handle.from_path(path), handle) - - def test_handle_attrs_readonly(self): - handle = Handle(None, 'kind', 'key') - with self.assertRaises(AttributeError): - handle.parent = 'foo' - with self.assertRaises(AttributeError): - handle.kind = 'foo' - with self.assertRaises(AttributeError): - handle.key = 'foo' - with self.assertRaises(AttributeError): - handle.path = 'foo' - - def test_restore_unknown(self): - framework = self.create_framework() - - class Foo(Object): - pass - - handle = Handle(None, "a_foo", "some_key") - - framework.register_type(Foo, None, handle.kind) - - try: - framework.load_snapshot(handle) - except NoSnapshotError as e: - self.assertEqual(e.handle_path, str(handle)) - self.assertEqual(str(e), "no snapshot data found for a_foo[some_key] object") - else: - self.fail("exception NoSnapshotError not raised") - - def test_snapshot_roundtrip(self): - class Foo: - def __init__(self, handle, n): - self.handle = handle - self.my_n = n - - def snapshot(self): - return {"My N!": self.my_n} - - def restore(self, snapshot): - self.my_n = snapshot["My N!"] + 1 - - handle = Handle(None, "a_foo", "some_key") - event = Foo(handle, 1) - - framework1 = self.create_framework() - framework1.register_type(Foo, None, handle.kind) - framework1.save_snapshot(event) - framework1.commit() - framework1.close() - - framework2 = self.create_framework() - framework2.register_type(Foo, None, handle.kind) - event2 = framework2.load_snapshot(handle) - self.assertEqual(event2.my_n, 2) - - framework2.save_snapshot(event2) - del event2 - gc.collect() - event3 = framework2.load_snapshot(handle) - self.assertEqual(event3.my_n, 3) - - framework2.drop_snapshot(event.handle) - framework2.commit() - framework2.close() - - framework3 = self.create_framework() - framework3.register_type(Foo, None, handle.kind) - - self.assertRaises(NoSnapshotError, framework3.load_snapshot, handle) - - def test_simple_event_observer(self): - framework = self.create_framework() - - class MyEvent(EventBase): - pass - - class MyNotifier(Object): - foo = EventSource(MyEvent) - bar = EventSource(MyEvent) - baz = EventSource(MyEvent) - - class MyObserver(Object): - def __init__(self, parent, key): - super().__init__(parent, key) - self.seen = [] - - def on_any(self, event): - self.seen.append("on_any:" + event.handle.kind) - - def on_foo(self, event): - self.seen.append("on_foo:" + event.handle.kind) - - pub = MyNotifier(framework, "1") - obs = MyObserver(framework, "1") - - framework.observe(pub.foo, obs.on_any) - framework.observe(pub.bar, obs.on_any) - framework.observe(pub.foo, obs) # Method name defaults to on_. - - try: - framework.observe(pub.baz, obs) - except RuntimeError as e: - self.assertEqual( - str(e), - 'Observer method not provided explicitly' - ' and MyObserver type has no "on_baz" method') - else: - self.fail("RuntimeError not raised") - - pub.foo.emit() - pub.bar.emit() - - self.assertEqual(obs.seen, ["on_any:foo", "on_foo:foo", "on_any:bar"]) - - def test_bad_sig_observer(self): - - class MyEvent(EventBase): - pass - - class MyNotifier(Object): - foo = EventSource(MyEvent) - bar = EventSource(MyEvent) - baz = EventSource(MyEvent) - qux = EventSource(MyEvent) - - class MyObserver(Object): - def on_foo(self): - assert False, 'should not be reached' - - def on_bar(self, event, extra): - assert False, 'should not be reached' - - def on_baz(self, event, extra=None, *, k): - assert False, 'should not be reached' - - def on_qux(self, event, extra=None): - assert False, 'should not be reached' - - framework = self.create_framework() - pub = MyNotifier(framework, "pub") - obs = MyObserver(framework, "obs") - - with self.assertRaises(TypeError): - framework.observe(pub.foo, obs) - with self.assertRaises(TypeError): - framework.observe(pub.bar, obs) - with self.assertRaises(TypeError): - framework.observe(pub.baz, obs) - framework.observe(pub.qux, obs) - - def test_on_pre_commit_emitted(self): - framework = self.create_framework() - - class PreCommitObserver(Object): - - _stored = StoredState() - - def __init__(self, parent, key): - super().__init__(parent, key) - self.seen = [] - self._stored.myinitdata = 40 - - def on_pre_commit(self, event): - self._stored.myinitdata = 41 - self._stored.mydata = 42 - self.seen.append(type(event)) - - def on_commit(self, event): - # Modifications made here will not be persisted. - self._stored.myinitdata = 42 - self._stored.mydata = 43 - self._stored.myotherdata = 43 - self.seen.append(type(event)) - - obs = PreCommitObserver(framework, None) - - framework.observe(framework.on.pre_commit, obs.on_pre_commit) - - framework.commit() - - self.assertEqual(obs._stored.myinitdata, 41) - self.assertEqual(obs._stored.mydata, 42) - self.assertTrue(obs.seen, [PreCommitEvent, CommitEvent]) - framework.close() - - other_framework = self.create_framework() - - new_obs = PreCommitObserver(other_framework, None) - - self.assertEqual(obs._stored.myinitdata, 41) - self.assertEqual(new_obs._stored.mydata, 42) - - with self.assertRaises(AttributeError): - new_obs._stored.myotherdata - - def test_defer_and_reemit(self): - framework = self.create_framework() - - class MyEvent(EventBase): - pass - - class MyNotifier1(Object): - a = EventSource(MyEvent) - b = EventSource(MyEvent) - - class MyNotifier2(Object): - c = EventSource(MyEvent) - - class MyObserver(Object): - def __init__(self, parent, key): - super().__init__(parent, key) - self.seen = [] - self.done = {} - - def on_any(self, event): - self.seen.append(event.handle.kind) - if not self.done.get(event.handle.kind): - event.defer() - - pub1 = MyNotifier1(framework, "1") - pub2 = MyNotifier2(framework, "1") - obs1 = MyObserver(framework, "1") - obs2 = MyObserver(framework, "2") - - framework.observe(pub1.a, obs1.on_any) - framework.observe(pub1.b, obs1.on_any) - framework.observe(pub1.a, obs2.on_any) - framework.observe(pub1.b, obs2.on_any) - framework.observe(pub2.c, obs2.on_any) - - pub1.a.emit() - pub1.b.emit() - pub2.c.emit() - - # Events remain stored because they were deferred. - ev_a_handle = Handle(pub1, "a", "1") - framework.load_snapshot(ev_a_handle) - ev_b_handle = Handle(pub1, "b", "2") - framework.load_snapshot(ev_b_handle) - ev_c_handle = Handle(pub2, "c", "3") - framework.load_snapshot(ev_c_handle) - # make sure the objects are gone before we reemit them - gc.collect() - - framework.reemit() - obs1.done["a"] = True - obs2.done["b"] = True - framework.reemit() - framework.reemit() - obs1.done["b"] = True - obs2.done["a"] = True - framework.reemit() - obs2.done["c"] = True - framework.reemit() - framework.reemit() - framework.reemit() - - self.assertEqual(" ".join(obs1.seen), "a b a b a b b b") - self.assertEqual(" ".join(obs2.seen), "a b c a b c a b c a c a c c") - - # Now the event objects must all be gone from storage. - self.assertRaises(NoSnapshotError, framework.load_snapshot, ev_a_handle) - self.assertRaises(NoSnapshotError, framework.load_snapshot, ev_b_handle) - self.assertRaises(NoSnapshotError, framework.load_snapshot, ev_c_handle) - - def test_custom_event_data(self): - framework = self.create_framework() - - class MyEvent(EventBase): - def __init__(self, handle, n): - super().__init__(handle) - self.my_n = n - - def snapshot(self): - return {"My N!": self.my_n} - - def restore(self, snapshot): - super().restore(snapshot) - self.my_n = snapshot["My N!"] + 1 - - class MyNotifier(Object): - foo = EventSource(MyEvent) - - class MyObserver(Object): - def __init__(self, parent, key): - super().__init__(parent, key) - self.seen = [] - - def on_foo(self, event): - self.seen.append("on_foo:{}={}".format(event.handle.kind, event.my_n)) - event.defer() - - pub = MyNotifier(framework, "1") - obs = MyObserver(framework, "1") - - framework.observe(pub.foo, obs) - - pub.foo.emit(1) - - framework.reemit() - - # Two things being checked here: - # - # 1. There's a restore roundtrip before the event is first observed. - # That means the data is safe before it's ever seen, and the - # roundtrip logic is tested under normal circumstances. - # - # 2. The renotification restores from the pristine event, not - # from the one modified during the first restore (otherwise - # we'd get a foo=3). - # - self.assertEqual(obs.seen, ["on_foo:foo=2", "on_foo:foo=2"]) - - def test_weak_observer(self): - framework = self.create_framework() - - observed_events = [] - - class MyEvent(EventBase): - pass - - class MyEvents(ObjectEvents): - foo = EventSource(MyEvent) - - class MyNotifier(Object): - on = MyEvents() - - class MyObserver(Object): - def on_foo(self, event): - observed_events.append("foo") - - pub = MyNotifier(framework, "1") - obs = MyObserver(framework, "2") - - framework.observe(pub.on.foo, obs) - pub.on.foo.emit() - self.assertEqual(observed_events, ["foo"]) - # Now delete the observer, and note that when we emit the event, it - # doesn't update the local slice again - del obs - gc.collect() - pub.on.foo.emit() - self.assertEqual(observed_events, ["foo"]) - - def test_forget_and_multiple_objects(self): - framework = self.create_framework() - - class MyObject(Object): - pass - - o1 = MyObject(framework, "path") - # Creating a second object at the same path should fail with RuntimeError - with self.assertRaises(RuntimeError): - o2 = MyObject(framework, "path") - # Unless we _forget the object first - framework._forget(o1) - o2 = MyObject(framework, "path") - self.assertEqual(o1.handle.path, o2.handle.path) - # Deleting the tracked object should also work - del o2 - gc.collect() - o3 = MyObject(framework, "path") - self.assertEqual(o1.handle.path, o3.handle.path) - framework.close() - # Or using a second framework - framework_copy = self.create_framework() - o_copy = MyObject(framework_copy, "path") - self.assertEqual(o1.handle.path, o_copy.handle.path) - - def test_forget_and_multiple_objects_with_load_snapshot(self): - framework = self.create_framework() - - class MyObject(Object): - def __init__(self, parent, name): - super().__init__(parent, name) - self.value = name - - def snapshot(self): - return self.value - - def restore(self, value): - self.value = value - - framework.register_type(MyObject, None, MyObject.handle_kind) - o1 = MyObject(framework, "path") - framework.save_snapshot(o1) - framework.commit() - o_handle = o1.handle - del o1 - gc.collect() - o2 = framework.load_snapshot(o_handle) - # Trying to load_snapshot a second object at the same path should fail with RuntimeError - with self.assertRaises(RuntimeError): - framework.load_snapshot(o_handle) - # Unless we _forget the object first - framework._forget(o2) - o3 = framework.load_snapshot(o_handle) - self.assertEqual(o2.value, o3.value) - # A loaded object also prevents direct creation of an object - with self.assertRaises(RuntimeError): - MyObject(framework, "path") - framework.close() - # But we can create an object, or load a snapshot in a copy of the framework - framework_copy1 = self.create_framework() - o_copy1 = MyObject(framework_copy1, "path") - self.assertEqual(o_copy1.value, "path") - framework_copy1.close() - framework_copy2 = self.create_framework() - framework_copy2.register_type(MyObject, None, MyObject.handle_kind) - o_copy2 = framework_copy2.load_snapshot(o_handle) - self.assertEqual(o_copy2.value, "path") - - def test_events_base(self): - framework = self.create_framework() - - class MyEvent(EventBase): - pass - - class MyEvents(ObjectEvents): - foo = EventSource(MyEvent) - bar = EventSource(MyEvent) - - class MyNotifier(Object): - on = MyEvents() - - class MyObserver(Object): - def __init__(self, parent, key): - super().__init__(parent, key) - self.seen = [] - - def on_foo(self, event): - self.seen.append("on_foo:{}".format(event.handle.kind)) - event.defer() - - def on_bar(self, event): - self.seen.append("on_bar:{}".format(event.handle.kind)) - - pub = MyNotifier(framework, "1") - obs = MyObserver(framework, "1") - - # Confirm that temporary persistence of BoundEvents doesn't cause errors, - # and that events can be observed. - for bound_event in [pub.on.foo, pub.on.bar]: - framework.observe(bound_event, obs) - - # Confirm that events can be emitted and seen. - pub.on.foo.emit() - - self.assertEqual(obs.seen, ["on_foo:foo"]) - - def test_conflicting_event_attributes(self): - class MyEvent(EventBase): - pass - - event = EventSource(MyEvent) - - class MyEvents(ObjectEvents): - foo = event - - with self.assertRaises(RuntimeError) as cm: - class OtherEvents(ObjectEvents): - foo = event - self.assertEqual( - str(cm.exception), - "EventSource(MyEvent) reused as MyEvents.foo and OtherEvents.foo") - - with self.assertRaises(RuntimeError) as cm: - class MyNotifier(Object): - on = MyEvents() - bar = event - self.assertEqual( - str(cm.exception), - "EventSource(MyEvent) reused as MyEvents.foo and MyNotifier.bar") - - def test_reemit_ignores_unknown_event_type(self): - # The event type may have been gone for good, and nobody cares, - # so this shouldn't be an error scenario. - - framework = self.create_framework() - - class MyEvent(EventBase): - pass - - class MyNotifier(Object): - foo = EventSource(MyEvent) - - class MyObserver(Object): - def __init__(self, parent, key): - super().__init__(parent, key) - self.seen = [] - - def on_foo(self, event): - self.seen.append(event.handle) - event.defer() - - pub = MyNotifier(framework, "1") - obs = MyObserver(framework, "1") - - framework.observe(pub.foo, obs) - pub.foo.emit() - - event_handle = obs.seen[0] - self.assertEqual(event_handle.kind, "foo") - - framework.commit() - framework.close() - - framework_copy = self.create_framework() - - # No errors on missing event types here. - framework_copy.reemit() - - # Register the type and check that the event is gone from storage. - framework_copy.register_type(MyEvent, event_handle.parent, event_handle.kind) - self.assertRaises(NoSnapshotError, framework_copy.load_snapshot, event_handle) - - def test_auto_register_event_types(self): - framework = self.create_framework() - - class MyFoo(EventBase): - pass - - class MyBar(EventBase): - pass - - class MyEvents(ObjectEvents): - foo = EventSource(MyFoo) - - class MyNotifier(Object): - on = MyEvents() - bar = EventSource(MyBar) - - class MyObserver(Object): - def __init__(self, parent, key): - super().__init__(parent, key) - self.seen = [] - - def on_foo(self, event): - self.seen.append("on_foo:{}:{}".format(type(event).__name__, event.handle.kind)) - event.defer() - - def on_bar(self, event): - self.seen.append("on_bar:{}:{}".format(type(event).__name__, event.handle.kind)) - event.defer() - - pub = MyNotifier(framework, "1") - obs = MyObserver(framework, "1") - - pub.on.foo.emit() - pub.bar.emit() - - framework.observe(pub.on.foo, obs) - framework.observe(pub.bar, obs) - - pub.on.foo.emit() - pub.bar.emit() - - self.assertEqual(obs.seen, ["on_foo:MyFoo:foo", "on_bar:MyBar:bar"]) - - def test_dynamic_event_types(self): - framework = self.create_framework() - - class MyEventsA(ObjectEvents): - handle_kind = 'on_a' - - class MyEventsB(ObjectEvents): - handle_kind = 'on_b' - - class MyNotifier(Object): - on_a = MyEventsA() - on_b = MyEventsB() - - class MyObserver(Object): - def __init__(self, parent, key): - super().__init__(parent, key) - self.seen = [] - - def on_foo(self, event): - self.seen.append("on_foo:{}:{}".format(type(event).__name__, event.handle.kind)) - event.defer() - - def on_bar(self, event): - self.seen.append("on_bar:{}:{}".format(type(event).__name__, event.handle.kind)) - event.defer() - - pub = MyNotifier(framework, "1") - obs = MyObserver(framework, "1") - - class MyFoo(EventBase): - pass - - class MyBar(EventBase): - pass - - class DeadBeefEvent(EventBase): - pass - - class NoneEvent(EventBase): - pass - - pub.on_a.define_event("foo", MyFoo) - pub.on_b.define_event("bar", MyBar) - - framework.observe(pub.on_a.foo, obs) - framework.observe(pub.on_b.bar, obs) - - pub.on_a.foo.emit() - pub.on_b.bar.emit() - - self.assertEqual(obs.seen, ["on_foo:MyFoo:foo", "on_bar:MyBar:bar"]) - - # Definitions remained local to the specific type. - self.assertRaises(AttributeError, lambda: pub.on_a.bar) - self.assertRaises(AttributeError, lambda: pub.on_b.foo) - - # Try to use an event name which is not a valid python identifier. - with self.assertRaises(RuntimeError): - pub.on_a.define_event("dead-beef", DeadBeefEvent) - - # Try to use a python keyword for an event name. - with self.assertRaises(RuntimeError): - pub.on_a.define_event("None", NoneEvent) - - # Try to override an existing attribute. - with self.assertRaises(RuntimeError): - pub.on_a.define_event("foo", MyFoo) - - def test_event_key_roundtrip(self): - class MyEvent(EventBase): - def __init__(self, handle, value): - super().__init__(handle) - self.value = value - - def snapshot(self): - return self.value - - def restore(self, value): - self.value = value - - class MyNotifier(Object): - foo = EventSource(MyEvent) - - class MyObserver(Object): - has_deferred = False - - def __init__(self, parent, key): - super().__init__(parent, key) - self.seen = [] - - def on_foo(self, event): - self.seen.append((event.handle.key, event.value)) - # Only defer the first event and once. - if not MyObserver.has_deferred: - event.defer() - MyObserver.has_deferred = True - - framework1 = self.create_framework() - pub1 = MyNotifier(framework1, "pub") - obs1 = MyObserver(framework1, "obs") - framework1.observe(pub1.foo, obs1) - pub1.foo.emit('first') - self.assertEqual(obs1.seen, [('1', 'first')]) - - framework1.commit() - framework1.close() - del framework1 - - framework2 = self.create_framework() - pub2 = MyNotifier(framework2, "pub") - obs2 = MyObserver(framework2, "obs") - framework2.observe(pub2.foo, obs2) - pub2.foo.emit('second') - framework2.reemit() - - # First observer didn't get updated, since framework it was bound to is gone. - self.assertEqual(obs1.seen, [('1', 'first')]) - # Second observer saw the new event plus the reemit of the first event. - # (The event key goes up by 2 due to the pre-commit and commit events.) - self.assertEqual(obs2.seen, [('4', 'second'), ('1', 'first')]) - - def test_helper_properties(self): - framework = self.create_framework() - framework.model = 'test-model' - framework.meta = 'test-meta' - - my_obj = Object(framework, 'my_obj') - self.assertEqual(my_obj.model, framework.model) - - def test_ban_concurrent_frameworks(self): - f = self.create_framework() - with self.assertRaises(Exception) as cm: - self.create_framework() - self.assertIn('database is locked', str(cm.exception)) - f.close() - - def test_snapshot_saving_restricted_to_simple_types(self): - # this can not be saved, as it has not simple types! - to_be_saved = {"bar": TestFramework} - - class FooEvent(EventBase): - def snapshot(self): - return to_be_saved - - handle = Handle(None, "a_foo", "some_key") - event = FooEvent(handle) - - framework = self.create_framework() - framework.register_type(FooEvent, None, handle.kind) - with self.assertRaises(ValueError) as cm: - framework.save_snapshot(event) - expected = ( - "unable to save the data for FooEvent, it must contain only simple types: " - "{'bar': }") - self.assertEqual(str(cm.exception), expected) - - -class TestStoredState(unittest.TestCase): - - def setUp(self): - self.tmpdir = Path(tempfile.mkdtemp()) - self.addCleanup(shutil.rmtree, str(self.tmpdir)) - - def create_framework(self, cls=Framework): - framework = cls(self.tmpdir / "framework.data", self.tmpdir, None, None) - self.addCleanup(framework.close) - return framework - - def test_basic_state_storage(self): - class SomeObject(Object): - _stored = StoredState() - - self._stored_state_tests(SomeObject) - - def test_straight_subclass(self): - class SomeObject(Object): - _stored = StoredState() - - class Sub(SomeObject): - pass - - self._stored_state_tests(Sub) - - def test_straight_sub_subclass(self): - class SomeObject(Object): - _stored = StoredState() - - class Sub(SomeObject): - pass - - class SubSub(SomeObject): - pass - - self._stored_state_tests(SubSub) - - def test_two_subclasses(self): - class SomeObject(Object): - _stored = StoredState() - - class SubA(SomeObject): - pass - - class SubB(SomeObject): - pass - - self._stored_state_tests(SubA) - self._stored_state_tests(SubB) - - def test_the_crazy_thing(self): - class NoState(Object): - pass - - class StatedObject(NoState): - _stored = StoredState() - - class Sibling(NoState): - pass - - class FinalChild(StatedObject, Sibling): - pass - - self._stored_state_tests(FinalChild) - - def _stored_state_tests(self, cls): - framework = self.create_framework() - obj = cls(framework, "1") - - try: - obj._stored.foo - except AttributeError as e: - self.assertEqual(str(e), "attribute 'foo' is not stored") - else: - self.fail("AttributeError not raised") - - try: - obj._stored.on = "nonono" - except AttributeError as e: - self.assertEqual(str(e), "attribute 'on' is reserved and cannot be set") - else: - self.fail("AttributeError not raised") - - obj._stored.foo = 41 - obj._stored.foo = 42 - obj._stored.bar = "s" - obj._stored.baz = 4.2 - obj._stored.bing = True - - self.assertEqual(obj._stored.foo, 42) - - framework.commit() - - # This won't be committed, and should not be seen. - obj._stored.foo = 43 - - framework.close() - - # Since this has the same absolute object handle, it will get its state back. - framework_copy = self.create_framework() - obj_copy = cls(framework_copy, "1") - self.assertEqual(obj_copy._stored.foo, 42) - self.assertEqual(obj_copy._stored.bar, "s") - self.assertEqual(obj_copy._stored.baz, 4.2) - self.assertEqual(obj_copy._stored.bing, True) - - framework_copy.close() - - def test_two_subclasses_no_conflicts(self): - class Base(Object): - _stored = StoredState() - - class SubA(Base): - pass - - class SubB(Base): - pass - - framework = self.create_framework() - a = SubA(framework, None) - b = SubB(framework, None) - z = Base(framework, None) - - a._stored.foo = 42 - b._stored.foo = "hello" - z._stored.foo = {1} - - framework.commit() - framework.close() - - framework2 = self.create_framework() - a2 = SubA(framework2, None) - b2 = SubB(framework2, None) - z2 = Base(framework2, None) - - self.assertEqual(a2._stored.foo, 42) - self.assertEqual(b2._stored.foo, "hello") - self.assertEqual(z2._stored.foo, {1}) - - def test_two_names_one_state(self): - class Mine(Object): - _stored = StoredState() - _stored2 = _stored - - framework = self.create_framework() - obj = Mine(framework, None) - - with self.assertRaises(RuntimeError): - obj._stored.foo = 42 - - with self.assertRaises(RuntimeError): - obj._stored2.foo = 42 - - framework.close() - - # make sure we're not changing the object on failure - self.assertNotIn("_stored", obj.__dict__) - self.assertNotIn("_stored2", obj.__dict__) - - def test_same_name_two_classes(self): - class Base(Object): - pass - - class A(Base): - _stored = StoredState() - - class B(Base): - _stored = A._stored - - framework = self.create_framework() - a = A(framework, None) - b = B(framework, None) - - # NOTE it's the second one that actually triggers the - # exception, but that's an implementation detail - a._stored.foo = 42 - - with self.assertRaises(RuntimeError): - b._stored.foo = "xyzzy" - - framework.close() - - # make sure we're not changing the object on failure - self.assertNotIn("_stored", b.__dict__) - - def test_mutable_types_invalid(self): - framework = self.create_framework() - - class SomeObject(Object): - _stored = StoredState() - - obj = SomeObject(framework, '1') - try: - class CustomObject: - pass - obj._stored.foo = CustomObject() - except AttributeError as e: - self.assertEqual( - str(e), - "attribute 'foo' cannot be a CustomObject: must be int/float/dict/list/etc") - else: - self.fail('AttributeError not raised') - - framework.commit() - - def test_mutable_types(self): - # Test and validation functions in a list of 2-tuples. - # Assignment and keywords like del are not supported in lambdas - # so functions are used instead. - test_operations = [( - lambda: {}, # Operand A. - None, # Operand B. - {}, # Expected result. - lambda a, b: None, # Operation to perform. - lambda res, expected_res: self.assertEqual(res, expected_res) # Validation to perform. - ), ( - lambda: {}, - {'a': {}}, - {'a': {}}, - lambda a, b: a.update(b), - lambda res, expected_res: self.assertEqual(res, expected_res) - ), ( - lambda: {'a': {}}, - {'b': 'c'}, - {'a': {'b': 'c'}}, - lambda a, b: a['a'].update(b), - lambda res, expected_res: self.assertEqual(res, expected_res) - ), ( - lambda: {'a': {'b': 'c'}}, - {'d': 'e'}, - {'a': {'b': 'c', 'd': 'e'}}, - lambda a, b: a['a'].update(b), - lambda res, expected_res: self.assertEqual(res, expected_res) - ), ( - lambda: {'a': {'b': 'c', 'd': 'e'}}, - 'd', - {'a': {'b': 'c'}}, - lambda a, b: a['a'].pop(b), - lambda res, expected_res: self.assertEqual(res, expected_res) - ), ( - lambda: {'s': set()}, - 'a', - {'s': {'a'}}, - lambda a, b: a['s'].add(b), - lambda res, expected_res: self.assertEqual(res, expected_res) - ), ( - lambda: {'s': {'a'}}, - 'a', - {'s': set()}, - lambda a, b: a['s'].discard(b), - lambda res, expected_res: self.assertEqual(res, expected_res) - ), ( - lambda: [], - None, - [], - lambda a, b: None, - lambda res, expected_res: self.assertEqual(res, expected_res) - ), ( - lambda: [], - 'a', - ['a'], - lambda a, b: a.append(b), - lambda res, expected_res: self.assertEqual(res, expected_res) - ), ( - lambda: ['a'], - ['c'], - ['a', ['c']], - lambda a, b: a.append(b), - lambda res, expected_res: ( - self.assertEqual(res, expected_res), - self.assertIsInstance(res[1], StoredList), - ) - ), ( - lambda: ['a', ['c']], - 'b', - ['b', 'a', ['c']], - lambda a, b: a.insert(0, b), - lambda res, expected_res: self.assertEqual(res, expected_res) - ), ( - lambda: ['b', 'a', ['c']], - ['d'], - ['b', ['d'], 'a', ['c']], - lambda a, b: a.insert(1, b), - lambda res, expected_res: ( - self.assertEqual(res, expected_res), - self.assertIsInstance(res[1], StoredList) - ), - ), ( - lambda: ['b', 'a', ['c']], - ['d'], - ['b', ['d'], ['c']], - # a[1] = b - lambda a, b: a.__setitem__(1, b), - lambda res, expected_res: ( - self.assertEqual(res, expected_res), - self.assertIsInstance(res[1], StoredList) - ), - ), ( - lambda: ['b', ['d'], 'a', ['c']], - 0, - [['d'], 'a', ['c']], - lambda a, b: a.pop(b), - lambda res, expected_res: self.assertEqual(res, expected_res) - ), ( - lambda: [['d'], 'a', ['c']], - ['d'], - ['a', ['c']], - lambda a, b: a.remove(b), - lambda res, expected_res: self.assertEqual(res, expected_res) - ), ( - lambda: ['a', ['c']], - 'd', - ['a', ['c', 'd']], - lambda a, b: a[1].append(b), - lambda res, expected_res: self.assertEqual(res, expected_res) - ), ( - lambda: ['a', ['c', 'd']], - 1, - ['a', ['c']], - lambda a, b: a[1].pop(b), - lambda res, expected_res: self.assertEqual(res, expected_res) - ), ( - lambda: ['a', ['c']], - 'd', - ['a', ['c', 'd']], - lambda a, b: a[1].insert(1, b), - lambda res, expected_res: self.assertEqual(res, expected_res) - ), ( - lambda: ['a', ['c', 'd']], - 'd', - ['a', ['c']], - lambda a, b: a[1].remove(b), - lambda res, expected_res: self.assertEqual(res, expected_res) - ), ( - lambda: set(), - None, - set(), - lambda a, b: None, - lambda res, expected_res: self.assertEqual(res, expected_res) - ), ( - lambda: set(), - 'a', - set(['a']), - lambda a, b: a.add(b), - lambda res, expected_res: self.assertEqual(res, expected_res) - ), ( - lambda: set(['a']), - 'a', - set(), - lambda a, b: a.discard(b), - lambda res, expected_res: self.assertEqual(res, expected_res) - ), ( - lambda: set(), - {'a'}, - set(), - # Nested sets are not allowed as sets themselves are not hashable. - lambda a, b: self.assertRaises(TypeError, a.add, b), - lambda res, expected_res: self.assertEqual(res, expected_res) - )] - - class SomeObject(Object): - _stored = StoredState() - - class WrappedFramework(Framework): - def __init__(self, data_path, charm_dir, meta, model): - super().__init__(data_path, charm_dir, meta, model) - self.snapshots = [] - - def save_snapshot(self, value): - if value.handle.path == 'SomeObject[1]/StoredStateData[_stored]': - self.snapshots.append((type(value), value.snapshot())) - return super().save_snapshot(value) - - # Validate correctness of modification operations. - for get_a, b, expected_res, op, validate_op in test_operations: - framework = self.create_framework(cls=WrappedFramework) - obj = SomeObject(framework, '1') - - obj._stored.a = get_a() - self.assertTrue(isinstance(obj._stored, BoundStoredState)) - - op(obj._stored.a, b) - validate_op(obj._stored.a, expected_res) - - obj._stored.a = get_a() - framework.commit() - # We should see an update for initializing a - self.assertEqual(framework.snapshots, [ - (StoredStateData, {'a': get_a()}), - ]) - del obj - gc.collect() - obj_copy1 = SomeObject(framework, '1') - self.assertEqual(obj_copy1._stored.a, get_a()) - - op(obj_copy1._stored.a, b) - validate_op(obj_copy1._stored.a, expected_res) - framework.commit() - framework.close() - - framework_copy = self.create_framework(cls=WrappedFramework) - - obj_copy2 = SomeObject(framework_copy, '1') - - validate_op(obj_copy2._stored.a, expected_res) - - # Commit saves the pre-commit and commit events, and the framework - # event counter, but shouldn't update the stored state of my object - framework.snapshots.clear() - framework_copy.commit() - self.assertEqual(framework_copy.snapshots, []) - framework_copy.close() - - def test_comparison_operations(self): - test_operations = [( - {"1"}, # Operand A. - {"1", "2"}, # Operand B. - lambda a, b: a < b, # Operation to test. - True, # Result of op(A, B). - False, # Result of op(B, A). - ), ( - {"1"}, - {"1", "2"}, - lambda a, b: a > b, - False, - True - ), ( - # Empty set comparison. - set(), - set(), - lambda a, b: a == b, - True, - True - ), ( - {"a", "c"}, - {"c", "a"}, - lambda a, b: a == b, - True, - True - ), ( - dict(), - dict(), - lambda a, b: a == b, - True, - True - ), ( - {"1": "2"}, - {"1": "2"}, - lambda a, b: a == b, - True, - True - ), ( - {"1": "2"}, - {"1": "3"}, - lambda a, b: a == b, - False, - False - ), ( - [], - [], - lambda a, b: a == b, - True, - True - ), ( - [1, 2], - [1, 2], - lambda a, b: a == b, - True, - True - ), ( - [1, 2, 5, 6], - [1, 2, 5, 8, 10], - lambda a, b: a <= b, - True, - False - ), ( - [1, 2, 5, 6], - [1, 2, 5, 8, 10], - lambda a, b: a < b, - True, - False - ), ( - [1, 2, 5, 8], - [1, 2, 5, 6, 10], - lambda a, b: a > b, - True, - False - ), ( - [1, 2, 5, 8], - [1, 2, 5, 6, 10], - lambda a, b: a >= b, - True, - False - )] - - class SomeObject(Object): - _stored = StoredState() - - framework = self.create_framework() - - for i, (a, b, op, op_ab, op_ba) in enumerate(test_operations): - obj = SomeObject(framework, str(i)) - obj._stored.a = a - self.assertEqual(op(obj._stored.a, b), op_ab) - self.assertEqual(op(b, obj._stored.a), op_ba) - - def test_set_operations(self): - test_operations = [( - {"1"}, # A set to test an operation against (other_set). - lambda a, b: a | b, # An operation to test. - {"1", "a", "b"}, # The expected result of operation(obj._stored.set, other_set). - {"1", "a", "b"} # The expected result of operation(other_set, obj._stored.set). - ), ( - {"a", "c"}, - lambda a, b: a - b, - {"b"}, - {"c"} - ), ( - {"a", "c"}, - lambda a, b: a & b, - {"a"}, - {"a"} - ), ( - {"a", "c", "d"}, - lambda a, b: a ^ b, - {"b", "c", "d"}, - {"b", "c", "d"} - ), ( - set(), - lambda a, b: set(a), - {"a", "b"}, - set() - )] - - class SomeObject(Object): - _stored = StoredState() - - framework = self.create_framework() - - # Validate that operations between StoredSet and built-in sets - # only result in built-in sets being returned. - # Make sure that commutativity is preserved and that the - # original sets are not changed or used as a result. - for i, (variable_operand, operation, ab_res, ba_res) in enumerate(test_operations): - obj = SomeObject(framework, str(i)) - obj._stored.set = {"a", "b"} - - for a, b, expected in [ - (obj._stored.set, variable_operand, ab_res), - (variable_operand, obj._stored.set, ba_res)]: - old_a = set(a) - old_b = set(b) - - result = operation(a, b) - self.assertEqual(result, expected) - - # Common sanity checks - self.assertIsNot(obj._stored.set._under, result) - self.assertIsNot(result, a) - self.assertIsNot(result, b) - self.assertEqual(a, old_a) - self.assertEqual(b, old_b) - - def test_set_default(self): - framework = self.create_framework() - - class StatefulObject(Object): - _stored = StoredState() - parent = StatefulObject(framework, 'key') - parent._stored.set_default(foo=1) - self.assertEqual(parent._stored.foo, 1) - parent._stored.set_default(foo=2) - # foo was already set, so it doesn't get replaced - self.assertEqual(parent._stored.foo, 1) - parent._stored.set_default(foo=3, bar=4) - self.assertEqual(parent._stored.foo, 1) - self.assertEqual(parent._stored.bar, 4) - # reloading the state still leaves things at the default values - framework.commit() - del parent - parent = StatefulObject(framework, 'key') - parent._stored.set_default(foo=5, bar=6) - self.assertEqual(parent._stored.foo, 1) - self.assertEqual(parent._stored.bar, 4) - # TODO: jam 2020-01-30 is there a clean way to tell that - # parent._stored._data.dirty is False? - - -def create_model(testcase): - """Create a Model object.""" - unit_name = 'myapp/0' - patcher = patch.dict(os.environ, {'JUJU_UNIT_NAME': unit_name}) - patcher.start() - testcase.addCleanup(patcher.stop) - - backend = model.ModelBackend() - meta = charm.CharmMeta() - test_model = model.Model('myapp/0', meta, backend) - return test_model - - -def create_framework(testcase, model=None): - """Create a Framework object.""" - framework = Framework(":memory:", charm_dir='non-existant', meta=None, model=model) - testcase.addCleanup(framework.close) - return framework - - -class GenericObserver(Object): - """Generic observer for the tests.""" - - def __init__(self, parent, key): - super().__init__(parent, key) - self.called = False - - def callback_method(self, event): - """Set the instance .called to True.""" - self.called = True - - -@patch('sys.stderr', new_callable=io.StringIO) -class BreakpointTests(unittest.TestCase): - - def test_ignored(self, fake_stderr): - # It doesn't do anything really unless proper environment is there. - with patch.dict(os.environ): - os.environ.pop('JUJU_DEBUG_AT', None) - framework = create_framework(self) - - with patch('pdb.Pdb.set_trace') as mock: - framework.breakpoint() - self.assertEqual(mock.call_count, 0) - self.assertEqual(fake_stderr.getvalue(), "") - - def test_pdb_properly_called(self, fake_stderr): - # The debugger needs to leave the user in the frame where the breakpoint is executed, - # which for the test is the frame we're calling it here in the test :). - with patch.dict(os.environ, {'JUJU_DEBUG_AT': 'all'}): - framework = create_framework(self) - - with patch('pdb.Pdb.set_trace') as mock: - this_frame = inspect.currentframe() - framework.breakpoint() - - self.assertEqual(mock.call_count, 1) - self.assertEqual(mock.call_args, ((this_frame,), {})) - - def test_welcome_message(self, fake_stderr): - # Check that an initial message is shown to the user when code is interrupted. - with patch.dict(os.environ, {'JUJU_DEBUG_AT': 'all'}): - framework = create_framework(self) - with patch('pdb.Pdb.set_trace'): - framework.breakpoint() - self.assertEqual(fake_stderr.getvalue(), _BREAKPOINT_WELCOME_MESSAGE) - - def test_welcome_message_not_multiple(self, fake_stderr): - # Check that an initial message is NOT shown twice if the breakpoint is exercised - # twice in the same run. - with patch.dict(os.environ, {'JUJU_DEBUG_AT': 'all'}): - framework = create_framework(self) - with patch('pdb.Pdb.set_trace'): - framework.breakpoint() - self.assertEqual(fake_stderr.getvalue(), _BREAKPOINT_WELCOME_MESSAGE) - framework.breakpoint() - self.assertEqual(fake_stderr.getvalue(), _BREAKPOINT_WELCOME_MESSAGE) - - def test_builtin_breakpoint_hooked(self, fake_stderr): - # Verify that the proper hook is set. - with patch.dict(os.environ, {'JUJU_DEBUG_AT': 'all'}): - create_framework(self) # creating the framework setups the hook - with patch('pdb.Pdb.set_trace') as mock: - # Calling through sys, not breakpoint() directly, so we can run the - # tests with Py < 3.7. - sys.breakpointhook() - self.assertEqual(mock.call_count, 1) - - def test_breakpoint_names(self, fake_stderr): - framework = create_framework(self) - - # Name rules: - # - must start and end with lowercase alphanumeric characters - # - only contain lowercase alphanumeric characters, or the hyphen "-" - good_names = [ - 'foobar', - 'foo-bar-baz', - 'foo-------bar', - 'foo123', - '778', - '77-xx', - 'a-b', - 'ab', - 'x', - ] - for name in good_names: - with self.subTest(name=name): - framework.breakpoint(name) - - bad_names = [ - '', - '.', - '-', - '...foo', - 'foo.bar', - 'bar--' - 'FOO', - 'FooBar', - 'foo bar', - 'foo_bar', - '/foobar', - 'break-here-☚', - ] - msg = 'breakpoint names must look like "foo" or "foo-bar"' - for name in bad_names: - with self.subTest(name=name): - with self.assertRaises(ValueError) as cm: - framework.breakpoint(name) - self.assertEqual(str(cm.exception), msg) - - reserved_names = [ - 'all', - 'hook', - ] - msg = 'breakpoint names "all" and "hook" are reserved' - for name in reserved_names: - with self.subTest(name=name): - with self.assertRaises(ValueError) as cm: - framework.breakpoint(name) - self.assertEqual(str(cm.exception), msg) - - not_really_names = [ - 123, - 1.1, - False, - ] - for name in not_really_names: - with self.subTest(name=name): - with self.assertRaises(TypeError) as cm: - framework.breakpoint(name) - self.assertEqual(str(cm.exception), 'breakpoint names must be strings') - - def check_trace_set(self, envvar_value, breakpoint_name, call_count): - """Helper to check the diverse combinations of situations.""" - with patch.dict(os.environ, {'JUJU_DEBUG_AT': envvar_value}): - framework = create_framework(self) - with patch('pdb.Pdb.set_trace') as mock: - framework.breakpoint(breakpoint_name) - self.assertEqual(mock.call_count, call_count) - - def test_unnamed_indicated_all(self, fake_stderr): - # If 'all' is indicated, unnamed breakpoints will always activate. - self.check_trace_set('all', None, 1) - - def test_unnamed_indicated_hook(self, fake_stderr): - # Special value 'hook' was indicated, nothing to do with any call. - self.check_trace_set('hook', None, 0) - - def test_named_indicated_specifically(self, fake_stderr): - # Some breakpoint was indicated, and the framework call used exactly that name. - self.check_trace_set('mybreak', 'mybreak', 1) - - def test_named_indicated_somethingelse(self, fake_stderr): - # Some breakpoint was indicated, but the framework call was not with that name. - self.check_trace_set('some-breakpoint', None, 0) - - def test_named_indicated_ingroup(self, fake_stderr): - # A multiple breakpoint was indicated, and the framework call used a name among those. - self.check_trace_set('some,mybreak,foobar', 'mybreak', 1) - - def test_named_indicated_all(self, fake_stderr): - # The framework indicated 'all', which includes any named breakpoint set. - self.check_trace_set('all', 'mybreak', 1) - - def test_named_indicated_hook(self, fake_stderr): - # The framework indicated the special value 'hook', nothing to do with any named call. - self.check_trace_set('hook', 'mybreak', 0) - - -class DebugHookTests(unittest.TestCase): - - def test_envvar_parsing_missing(self): - with patch.dict(os.environ): - os.environ.pop('JUJU_DEBUG_AT', None) - framework = create_framework(self) - self.assertEqual(framework._juju_debug_at, ()) - - def test_envvar_parsing_empty(self): - with patch.dict(os.environ, {'JUJU_DEBUG_AT': ''}): - framework = create_framework(self) - self.assertEqual(framework._juju_debug_at, ()) - - def test_envvar_parsing_simple(self): - with patch.dict(os.environ, {'JUJU_DEBUG_AT': 'hook'}): - framework = create_framework(self) - self.assertEqual(framework._juju_debug_at, ['hook']) - - def test_envvar_parsing_multiple(self): - with patch.dict(os.environ, {'JUJU_DEBUG_AT': 'foo,bar,all'}): - framework = create_framework(self) - self.assertEqual(framework._juju_debug_at, ['foo', 'bar', 'all']) - - def test_basic_interruption_enabled(self): - framework = create_framework(self) - framework._juju_debug_at = ['hook'] - - publisher = charm.CharmEvents(framework, "1") - observer = GenericObserver(framework, "1") - framework.observe(publisher.install, observer.callback_method) - - with patch('sys.stderr', new_callable=io.StringIO) as fake_stderr: - with patch('pdb.runcall') as mock: - publisher.install.emit() - - # Check that the pdb module was used correctly and that the callback method was NOT - # called (as we intercepted the normal pdb behaviour! this is to check that the - # framework didn't call the callback directly) - self.assertEqual(mock.call_count, 1) - expected_callback, expected_event = mock.call_args[0] - self.assertEqual(expected_callback, observer.callback_method) - self.assertIsInstance(expected_event, EventBase) - self.assertFalse(observer.called) - - # Verify proper message was given to the user. - self.assertEqual(fake_stderr.getvalue(), _BREAKPOINT_WELCOME_MESSAGE) - - def test_actions_are_interrupted(self): - test_model = create_model(self) - framework = create_framework(self, model=test_model) - framework._juju_debug_at = ['hook'] - - class CustomEvents(ObjectEvents): - foobar_action = EventSource(charm.ActionEvent) - - publisher = CustomEvents(framework, "1") - observer = GenericObserver(framework, "1") - framework.observe(publisher.foobar_action, observer.callback_method) - fake_script(self, 'action-get', "echo {}") - - with patch('sys.stderr', new_callable=io.StringIO): - with patch('pdb.runcall') as mock: - with patch.dict(os.environ, {'JUJU_ACTION_NAME': 'foobar'}): - publisher.foobar_action.emit() - - self.assertEqual(mock.call_count, 1) - self.assertFalse(observer.called) - - def test_internal_events_not_interrupted(self): - class MyNotifier(Object): - """Generic notifier for the tests.""" - bar = EventSource(EventBase) - - framework = create_framework(self) - framework._juju_debug_at = ['hook'] - - publisher = MyNotifier(framework, "1") - observer = GenericObserver(framework, "1") - framework.observe(publisher.bar, observer.callback_method) - - with patch('pdb.runcall') as mock: - publisher.bar.emit() - - self.assertEqual(mock.call_count, 0) - self.assertTrue(observer.called) - - def test_envvar_mixed(self): - framework = create_framework(self) - framework._juju_debug_at = ['foo', 'hook', 'all', 'whatever'] - - publisher = charm.CharmEvents(framework, "1") - observer = GenericObserver(framework, "1") - framework.observe(publisher.install, observer.callback_method) - - with patch('sys.stderr', new_callable=io.StringIO): - with patch('pdb.runcall') as mock: - publisher.install.emit() - - self.assertEqual(mock.call_count, 1) - self.assertFalse(observer.called) - - def test_no_registered_method(self): - framework = create_framework(self) - framework._juju_debug_at = ['hook'] - - publisher = charm.CharmEvents(framework, "1") - observer = GenericObserver(framework, "1") - - with patch('pdb.runcall') as mock: - publisher.install.emit() - - self.assertEqual(mock.call_count, 0) - self.assertFalse(observer.called) - - def test_envvar_nohook(self): - framework = create_framework(self) - framework._juju_debug_at = ['something-else'] - - publisher = charm.CharmEvents(framework, "1") - observer = GenericObserver(framework, "1") - framework.observe(publisher.install, observer.callback_method) - - with patch.dict(os.environ, {'JUJU_DEBUG_AT': 'something-else'}): - with patch('pdb.runcall') as mock: - publisher.install.emit() - - self.assertEqual(mock.call_count, 0) - self.assertTrue(observer.called) - - def test_envvar_missing(self): - framework = create_framework(self) - framework._juju_debug_at = () - - publisher = charm.CharmEvents(framework, "1") - observer = GenericObserver(framework, "1") - framework.observe(publisher.install, observer.callback_method) - - with patch('pdb.runcall') as mock: - publisher.install.emit() - - self.assertEqual(mock.call_count, 0) - self.assertTrue(observer.called) - - def test_welcome_message_not_multiple(self): - framework = create_framework(self) - framework._juju_debug_at = ['hook'] - - publisher = charm.CharmEvents(framework, "1") - observer = GenericObserver(framework, "1") - framework.observe(publisher.install, observer.callback_method) - - with patch('sys.stderr', new_callable=io.StringIO) as fake_stderr: - with patch('pdb.runcall') as mock: - publisher.install.emit() - self.assertEqual(fake_stderr.getvalue(), _BREAKPOINT_WELCOME_MESSAGE) - publisher.install.emit() - self.assertEqual(fake_stderr.getvalue(), _BREAKPOINT_WELCOME_MESSAGE) - self.assertEqual(mock.call_count, 2) diff --git a/magma/squid_cnf/charms/squid/mod/operator/test/test_helpers.py b/magma/squid_cnf/charms/squid/mod/operator/test/test_helpers.py deleted file mode 100644 index 131681d0ee5f0f5529dde828c0739bd9b783d471..0000000000000000000000000000000000000000 --- a/magma/squid_cnf/charms/squid/mod/operator/test/test_helpers.py +++ /dev/null @@ -1,81 +0,0 @@ -# Copyright 2019 Canonical Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -import pathlib -import subprocess -import shutil -import tempfile -import unittest - - -def fake_script(test_case, name, content): - if not hasattr(test_case, 'fake_script_path'): - fake_script_path = tempfile.mkdtemp('-fake_script') - os.environ['PATH'] = '{}:{}'.format(fake_script_path, os.environ["PATH"]) - - def cleanup(): - shutil.rmtree(fake_script_path) - os.environ['PATH'] = os.environ['PATH'].replace(fake_script_path + ':', '') - - test_case.addCleanup(cleanup) - test_case.fake_script_path = pathlib.Path(fake_script_path) - - with (test_case.fake_script_path / name).open('wt') as f: - # Before executing the provided script, dump the provided arguments in calls.txt. - f.write('''#!/bin/bash -{ echo -n $(basename $0); printf ";%s" "$@"; echo; } >> $(dirname $0)/calls.txt -''' + content) - os.chmod(str(test_case.fake_script_path / name), 0o755) - - -def fake_script_calls(test_case, clear=False): - try: - with (test_case.fake_script_path / 'calls.txt').open('r+t') as f: - calls = [line.split(';') for line in f.read().splitlines()] - if clear: - f.truncate(0) - return calls - except FileNotFoundError: - return [] - - -class FakeScriptTest(unittest.TestCase): - - def test_fake_script_works(self): - fake_script(self, 'foo', 'echo foo runs') - fake_script(self, 'bar', 'echo bar runs') - output = subprocess.getoutput('foo a "b c "; bar "d e" f') - self.assertEqual(output, 'foo runs\nbar runs') - self.assertEqual(fake_script_calls(self), [ - ['foo', 'a', 'b c '], - ['bar', 'd e', 'f'], - ]) - - def test_fake_script_clear(self): - fake_script(self, 'foo', 'echo foo runs') - - output = subprocess.getoutput('foo a "b c"') - self.assertEqual(output, 'foo runs') - - self.assertEqual(fake_script_calls(self, clear=True), [['foo', 'a', 'b c']]) - - fake_script(self, 'bar', 'echo bar runs') - - output = subprocess.getoutput('bar "d e" f') - self.assertEqual(output, 'bar runs') - - self.assertEqual(fake_script_calls(self, clear=True), [['bar', 'd e', 'f']]) - - self.assertEqual(fake_script_calls(self, clear=True), []) diff --git a/magma/squid_cnf/charms/squid/mod/operator/test/test_infra.py b/magma/squid_cnf/charms/squid/mod/operator/test/test_infra.py deleted file mode 100644 index 9d269547207924078ac83f90d853611a371dabf0..0000000000000000000000000000000000000000 --- a/magma/squid_cnf/charms/squid/mod/operator/test/test_infra.py +++ /dev/null @@ -1,130 +0,0 @@ -# Copyright 2020 Canonical Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import io -import itertools -import os -import re -import subprocess -import sys -import tempfile -import unittest -from unittest.mock import patch - -import autopep8 -from flake8.api.legacy import get_style_guide - - -def get_python_filepaths(): - """Helper to retrieve paths of Python files.""" - python_paths = ['setup.py'] - for root in ['ops', 'test']: - for dirpath, dirnames, filenames in os.walk(root): - for filename in filenames: - if filename.endswith(".py"): - python_paths.append(os.path.join(dirpath, filename)) - return python_paths - - -class InfrastructureTests(unittest.TestCase): - - def test_pep8(self): - # verify all files are nicely styled - python_filepaths = get_python_filepaths() - style_guide = get_style_guide() - fake_stdout = io.StringIO() - with patch('sys.stdout', fake_stdout): - report = style_guide.check_files(python_filepaths) - - # if flake8 didnt' report anything, we're done - if report.total_errors == 0: - return - - # grab on which files we have issues - flake8_issues = fake_stdout.getvalue().split('\n') - broken_filepaths = {item.split(':')[0] for item in flake8_issues if item} - - # give hints to the developer on how files' style could be improved - options = autopep8.parse_args(['']) - options.aggressive = 1 - options.diff = True - options.max_line_length = 99 - - issues = [] - for filepath in broken_filepaths: - diff = autopep8.fix_file(filepath, options=options) - if diff: - issues.append(diff) - - report = ["Please fix files as suggested by autopep8:"] + issues - report += ["\n-- Original flake8 reports:"] + flake8_issues - self.fail("\n".join(report)) - - def test_quote_backslashes(self): - # ensure we're not using unneeded backslash to escape strings - issues = [] - for filepath in get_python_filepaths(): - with open(filepath, "rt", encoding="utf8") as fh: - for idx, line in enumerate(fh, 1): - if (r'\"' in line or r"\'" in line) and "NOQA" not in line: - issues.append((filepath, idx, line.rstrip())) - if issues: - msgs = ["{}:{:d}:{}".format(*issue) for issue in issues] - self.fail("Spurious backslashes found, please fix these quotings:\n" + "\n".join(msgs)) - - def test_ensure_copyright(self): - # all non-empty Python files must have a proper copyright somewhere in the first 5 lines - issues = [] - regex = re.compile(r"# Copyright \d\d\d\d(-\d\d\d\d)? Canonical Ltd.\n") - for filepath in get_python_filepaths(): - if os.stat(filepath).st_size == 0: - continue - - with open(filepath, "rt", encoding="utf8") as fh: - for line in itertools.islice(fh, 5): - if regex.match(line): - break - else: - issues.append(filepath) - if issues: - self.fail("Please add copyright headers to the following files:\n" + "\n".join(issues)) - - -class ImportersTestCase(unittest.TestCase): - - template = "from ops import {module_name}" - - def test_imports(self): - mod_names = [ - 'charm', - 'framework', - 'main', - 'model', - 'testing', - ] - - for name in mod_names: - with self.subTest(name=name): - self.check(name) - - def check(self, name): - """Helper function to run the test.""" - _, testfile = tempfile.mkstemp() - self.addCleanup(os.unlink, testfile) - - with open(testfile, 'wt', encoding='utf8') as fh: - fh.write(self.template.format(module_name=name)) - - proc = subprocess.run([sys.executable, testfile], env={'PYTHONPATH': os.getcwd()}) - self.assertEqual(proc.returncode, 0) diff --git a/magma/squid_cnf/charms/squid/mod/operator/test/test_jujuversion.py b/magma/squid_cnf/charms/squid/mod/operator/test/test_jujuversion.py deleted file mode 100755 index d19fd60045800c61378bcb0496fc79926bc71110..0000000000000000000000000000000000000000 --- a/magma/squid_cnf/charms/squid/mod/operator/test/test_jujuversion.py +++ /dev/null @@ -1,130 +0,0 @@ -#!/usr/bin/env python3 -# Copyright 2019 Canonical Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import unittest - -from ops.jujuversion import JujuVersion - - -class TestJujuVersion(unittest.TestCase): - - def test_parsing(self): - test_cases = [ - ("0.0.0", 0, 0, '', 0, 0), - ("0.0.2", 0, 0, '', 2, 0), - ("0.1.0", 0, 1, '', 0, 0), - ("0.2.3", 0, 2, '', 3, 0), - ("10.234.3456", 10, 234, '', 3456, 0), - ("10.234.3456.1", 10, 234, '', 3456, 1), - ("1.21-alpha12", 1, 21, 'alpha', 12, 0), - ("1.21-alpha1.34", 1, 21, 'alpha', 1, 34), - ("2.7", 2, 7, '', 0, 0) - ] - - for vs, major, minor, tag, patch, build in test_cases: - v = JujuVersion(vs) - self.assertEqual(v.major, major) - self.assertEqual(v.minor, minor) - self.assertEqual(v.tag, tag) - self.assertEqual(v.patch, patch) - self.assertEqual(v.build, build) - - def test_parsing_errors(self): - invalid_versions = [ - "xyz", - "foo.bar", - "foo.bar.baz", - "dead.beef.ca.fe", - "1234567890.2.1", # The major version is too long. - "0.2..1", # Two periods next to each other. - "1.21.alpha1", # Tag comes after period. - "1.21-alpha", # No patch number but a tag is present. - "1.21-alpha1beta", # Non-numeric string after the patch number. - "1.21-alpha-dev", # Tag duplication. - "1.21-alpha_dev3", # Underscore in a tag. - "1.21-alpha123dev3", # Non-numeric string after the patch number. - ] - for v in invalid_versions: - with self.assertRaises(RuntimeError): - JujuVersion(v) - - def test_equality(self): - test_cases = [ - ("1.0.0", "1.0.0", True), - ("01.0.0", "1.0.0", True), - ("10.0.0", "9.0.0", False), - ("1.0.0", "1.0.1", False), - ("1.0.1", "1.0.0", False), - ("1.0.0", "1.1.0", False), - ("1.1.0", "1.0.0", False), - ("1.0.0", "2.0.0", False), - ("1.2-alpha1", "1.2.0", False), - ("1.2-alpha2", "1.2-alpha1", False), - ("1.2-alpha2.1", "1.2-alpha2", False), - ("1.2-alpha2.2", "1.2-alpha2.1", False), - ("1.2-beta1", "1.2-alpha1", False), - ("1.2-beta1", "1.2-alpha2.1", False), - ("1.2-beta1", "1.2.0", False), - ("1.2.1", "1.2.0", False), - ("2.0.0", "1.0.0", False), - ("2.0.0.0", "2.0.0", True), - ("2.0.0.0", "2.0.0.0", True), - ("2.0.0.1", "2.0.0.0", False), - ("2.0.1.10", "2.0.0.0", False), - ] - - for a, b, expected in test_cases: - self.assertEqual(JujuVersion(a) == JujuVersion(b), expected) - self.assertEqual(JujuVersion(a) == b, expected) - - def test_comparison(self): - test_cases = [ - ("1.0.0", "1.0.0", False, True), - ("01.0.0", "1.0.0", False, True), - ("10.0.0", "9.0.0", False, False), - ("1.0.0", "1.0.1", True, True), - ("1.0.1", "1.0.0", False, False), - ("1.0.0", "1.1.0", True, True), - ("1.1.0", "1.0.0", False, False), - ("1.0.0", "2.0.0", True, True), - ("1.2-alpha1", "1.2.0", True, True), - ("1.2-alpha2", "1.2-alpha1", False, False), - ("1.2-alpha2.1", "1.2-alpha2", False, False), - ("1.2-alpha2.2", "1.2-alpha2.1", False, False), - ("1.2-beta1", "1.2-alpha1", False, False), - ("1.2-beta1", "1.2-alpha2.1", False, False), - ("1.2-beta1", "1.2.0", True, True), - ("1.2.1", "1.2.0", False, False), - ("2.0.0", "1.0.0", False, False), - ("2.0.0.0", "2.0.0", False, True), - ("2.0.0.0", "2.0.0.0", False, True), - ("2.0.0.1", "2.0.0.0", False, False), - ("2.0.1.10", "2.0.0.0", False, False), - ] - - for a, b, expected_strict, expected_weak in test_cases: - self.assertEqual(JujuVersion(a) < JujuVersion(b), expected_strict) - self.assertEqual(JujuVersion(a) <= JujuVersion(b), expected_weak) - self.assertEqual(JujuVersion(b) > JujuVersion(a), expected_strict) - self.assertEqual(JujuVersion(b) >= JujuVersion(a), expected_weak) - # Implicit conversion. - self.assertEqual(JujuVersion(a) < b, expected_strict) - self.assertEqual(JujuVersion(a) <= b, expected_weak) - self.assertEqual(b > JujuVersion(a), expected_strict) - self.assertEqual(b >= JujuVersion(a), expected_weak) - - -if __name__ == "__main__": - unittest.main() diff --git a/magma/squid_cnf/charms/squid/mod/operator/test/test_log.py b/magma/squid_cnf/charms/squid/mod/operator/test/test_log.py deleted file mode 100644 index b7f74d5c901ffb7833c1e5523f9f69a1959999e1..0000000000000000000000000000000000000000 --- a/magma/squid_cnf/charms/squid/mod/operator/test/test_log.py +++ /dev/null @@ -1,140 +0,0 @@ -#!/usr/bin/python3 - -# Copyright 2020 Canonical Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import io -import unittest -from unittest.mock import patch -import importlib - -import logging -import ops.log - - -class FakeModelBackend: - - def __init__(self): - self._calls = [] - - def calls(self, clear=False): - calls = self._calls - if clear: - self._calls = [] - return calls - - def juju_log(self, message, level): - self._calls.append((message, level)) - - -def reset_logging(): - logging.shutdown() - importlib.reload(logging) - - -class TestLogging(unittest.TestCase): - - def setUp(self): - self.backend = FakeModelBackend() - - reset_logging() - self.addCleanup(reset_logging) - - def test_default_logging(self): - ops.log.setup_root_logging(self.backend) - - logger = logging.getLogger() - self.assertEqual(logger.level, logging.DEBUG) - self.assertIsInstance(logger.handlers[0], ops.log.JujuLogHandler) - - test_cases = [( - lambda: logger.critical('critical'), [('CRITICAL', 'critical')] - ), ( - lambda: logger.error('error'), [('ERROR', 'error')] - ), ( - lambda: logger.warning('warning'), [('WARNING', 'warning')] - ), ( - lambda: logger.info('info'), [('INFO', 'info')] - ), ( - lambda: logger.debug('debug'), [('DEBUG', 'debug')] - )] - - for do, res in test_cases: - do() - calls = self.backend.calls(clear=True) - self.assertEqual(calls, res) - - def test_handler_filtering(self): - logger = logging.getLogger() - logger.setLevel(logging.INFO) - logger.addHandler(ops.log.JujuLogHandler(self.backend, logging.WARNING)) - logger.info('foo') - self.assertEqual(self.backend.calls(), []) - logger.warning('bar') - self.assertEqual(self.backend.calls(), [('WARNING', 'bar')]) - - def test_no_stderr_without_debug(self): - buffer = io.StringIO() - with patch('sys.stderr', buffer): - ops.log.setup_root_logging(self.backend, debug=False) - logger = logging.getLogger() - logger.debug('debug message') - logger.info('info message') - logger.warning('warning message') - logger.critical('critical message') - self.assertEqual( - self.backend.calls(), - [('DEBUG', 'debug message'), - ('INFO', 'info message'), - ('WARNING', 'warning message'), - ('CRITICAL', 'critical message'), - ]) - self.assertEqual(buffer.getvalue(), "") - - def test_debug_logging(self): - buffer = io.StringIO() - with patch('sys.stderr', buffer): - ops.log.setup_root_logging(self.backend, debug=True) - logger = logging.getLogger() - logger.debug('debug message') - logger.info('info message') - logger.warning('warning message') - logger.critical('critical message') - self.assertEqual( - self.backend.calls(), - [('DEBUG', 'debug message'), - ('INFO', 'info message'), - ('WARNING', 'warning message'), - ('CRITICAL', 'critical message'), - ]) - self.assertRegex( - buffer.getvalue(), - r"\d\d\d\d-\d\d-\d\d \d\d:\d\d:\d\d,\d\d\d DEBUG debug message\n" - r"\d\d\d\d-\d\d-\d\d \d\d:\d\d:\d\d,\d\d\d INFO info message\n" - r"\d\d\d\d-\d\d-\d\d \d\d:\d\d:\d\d,\d\d\d WARNING warning message\n" - r"\d\d\d\d-\d\d-\d\d \d\d:\d\d:\d\d,\d\d\d CRITICAL critical message\n" - ) - - def test_reduced_logging(self): - ops.log.setup_root_logging(self.backend) - logger = logging.getLogger() - logger.setLevel(logging.WARNING) - logger.debug('debug') - logger.info('info') - logger.warning('warning') - self.assertEqual(self.backend.calls(), [('WARNING', 'warning')]) - - -if __name__ == '__main__': - unittest.main() diff --git a/magma/squid_cnf/charms/squid/mod/operator/test/test_main.py b/magma/squid_cnf/charms/squid/mod/operator/test/test_main.py deleted file mode 100755 index 0703b88aa035baa5e907cf07fc4c5f231ca5669f..0000000000000000000000000000000000000000 --- a/magma/squid_cnf/charms/squid/mod/operator/test/test_main.py +++ /dev/null @@ -1,675 +0,0 @@ -#!/usr/bin/env python3 -# Copyright 2019 Canonical Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import abc -import base64 -import logging -import os -import pickle -import shutil -import subprocess -import sys -import tempfile -import unittest -import importlib.util -from pathlib import Path - -from ops.charm import ( - CharmBase, - CharmEvents, - HookEvent, - InstallEvent, - StartEvent, - ConfigChangedEvent, - UpgradeCharmEvent, - UpdateStatusEvent, - LeaderSettingsChangedEvent, - RelationJoinedEvent, - RelationChangedEvent, - RelationDepartedEvent, - RelationBrokenEvent, - RelationEvent, - StorageAttachedEvent, - ActionEvent, - CollectMetricsEvent, -) - -from .test_helpers import fake_script, fake_script_calls - -# This relies on the expected repository structure to find a path to -# source of the charm under test. -TEST_CHARM_DIR = Path(__file__ + '/../charms/test_main').resolve() - -logger = logging.getLogger(__name__) - - -class SymlinkTargetError(Exception): - pass - - -class EventSpec: - def __init__(self, event_type, event_name, env_var=None, - relation_id=None, remote_app=None, remote_unit=None, - charm_config=None): - self.event_type = event_type - self.event_name = event_name - self.env_var = env_var - self.relation_id = relation_id - self.remote_app = remote_app - self.remote_unit = remote_unit - self.charm_config = charm_config - - -class TestMain(abc.ABC): - - @abc.abstractmethod - def _setup_entry_point(self, directory, entry_point): - """Set up the given entry point in the given directory. - - If not using dispatch, that would be a symlink / - pointing at src/charm.py; if using dispatch that would be the dispatch - symlink. It could also not be a symlink... - """ - return NotImplemented - - @abc.abstractmethod - def _call_event(self, rel_path, env): - """Set up the environment and call (i.e. run) the given event.""" - return NotImplemented - - @abc.abstractmethod - def test_setup_event_links(self): - """Test auto-creation of symlinks caused by initial events. - - Depending on the combination of dispatch and non-dispatch, this should - be checking for the creation or the _lack_ of creation, as appropriate. - """ - return NotImplemented - - def setUp(self): - self._setup_charm_dir() - - _, tmp_file = tempfile.mkstemp() - self._state_file = Path(tmp_file) - self.addCleanup(self._state_file.unlink) - - # Relations events are defined dynamically and modify the class attributes. - # We use a subclass temporarily to prevent these side effects from leaking. - class TestCharmEvents(CharmEvents): - pass - CharmBase.on = TestCharmEvents() - - def cleanup(): - shutil.rmtree(str(self.JUJU_CHARM_DIR)) - CharmBase.on = CharmEvents() - self.addCleanup(cleanup) - - fake_script(self, 'juju-log', "exit 0") - - # set to something other than None for tests that care - self.stdout = None - self.stderr = None - - def _setup_charm_dir(self): - self.JUJU_CHARM_DIR = Path(tempfile.mkdtemp()) / 'test_main' - self.hooks_dir = self.JUJU_CHARM_DIR / 'hooks' - charm_path = str(self.JUJU_CHARM_DIR / 'src/charm.py') - self.charm_exec_path = os.path.relpath(charm_path, - str(self.hooks_dir)) - shutil.copytree(str(TEST_CHARM_DIR), str(self.JUJU_CHARM_DIR)) - - charm_spec = importlib.util.spec_from_file_location("charm", charm_path) - self.charm_module = importlib.util.module_from_spec(charm_spec) - charm_spec.loader.exec_module(self.charm_module) - - self._prepare_initial_hooks() - - def _prepare_initial_hooks(self): - initial_hooks = ('install', 'start', 'upgrade-charm', 'disks-storage-attached') - self.hooks_dir.mkdir() - for hook in initial_hooks: - self._setup_entry_point(self.hooks_dir, hook) - - def _prepare_actions(self): - actions_meta = ''' -foo-bar: - description: Foos the bar. - title: foo-bar - params: - foo-name: - type: string - description: A foo name to bar. - silent: - type: boolean - description: - default: false - required: - - foo-name -start: - description: Start the unit.''' - actions_dir_name = 'actions' - actions_meta_file = 'actions.yaml' - - with (self.JUJU_CHARM_DIR / actions_meta_file).open('w+t') as f: - f.write(actions_meta) - actions_dir = self.JUJU_CHARM_DIR / actions_dir_name - actions_dir.mkdir() - for action_name in ('start', 'foo-bar'): - self._setup_entry_point(actions_dir, action_name) - - def _read_and_clear_state(self): - state = None - if self._state_file.stat().st_size: - with self._state_file.open('r+b') as state_file: - state = pickle.load(state_file) - state_file.truncate(0) - return state - - def _simulate_event(self, event_spec): - env = { - 'PATH': "{}:{}".format(Path(__file__).parent / 'bin', os.environ['PATH']), - 'JUJU_CHARM_DIR': str(self.JUJU_CHARM_DIR), - 'JUJU_UNIT_NAME': 'test_main/0', - 'CHARM_CONFIG': event_spec.charm_config, - } - if issubclass(event_spec.event_type, RelationEvent): - rel_name = event_spec.event_name.split('_')[0] - env.update({ - 'JUJU_RELATION': rel_name, - 'JUJU_RELATION_ID': str(event_spec.relation_id), - }) - remote_app = event_spec.remote_app - # For juju < 2.7 app name is extracted from JUJU_REMOTE_UNIT. - if remote_app is not None: - env['JUJU_REMOTE_APP'] = remote_app - - remote_unit = event_spec.remote_unit - if remote_unit is None: - remote_unit = '' - - env['JUJU_REMOTE_UNIT'] = remote_unit - else: - env.update({ - 'JUJU_REMOTE_UNIT': '', - 'JUJU_REMOTE_APP': '', - }) - if issubclass(event_spec.event_type, ActionEvent): - event_filename = event_spec.event_name[:-len('_action')].replace('_', '-') - env.update({ - event_spec.env_var: event_filename, - }) - if event_spec.env_var == 'JUJU_ACTION_NAME': - event_dir = 'actions' - else: - raise RuntimeError('invalid envar name specified for a action event') - else: - event_filename = event_spec.event_name.replace('_', '-') - event_dir = 'hooks' - - self._call_event(Path(event_dir, event_filename), env) - return self._read_and_clear_state() - - def test_event_reemitted(self): - # base64 encoding is used to avoid null bytes. - charm_config = base64.b64encode(pickle.dumps({ - 'STATE_FILE': self._state_file, - })) - - # First run "install" to make sure all hooks are set up. - state = self._simulate_event(EventSpec(InstallEvent, 'install', charm_config=charm_config)) - self.assertEqual(state['observed_event_types'], [InstallEvent]) - - state = self._simulate_event(EventSpec(ConfigChangedEvent, 'config-changed', - charm_config=charm_config)) - self.assertEqual(state['observed_event_types'], [ConfigChangedEvent]) - - # Re-emit should pick the deferred config-changed. - state = self._simulate_event(EventSpec(UpdateStatusEvent, 'update-status', - charm_config=charm_config)) - self.assertEqual(state['observed_event_types'], [ConfigChangedEvent, UpdateStatusEvent]) - - def test_no_reemission_on_collect_metrics(self): - # base64 encoding is used to avoid null bytes. - charm_config = base64.b64encode(pickle.dumps({ - 'STATE_FILE': self._state_file, - })) - fake_script(self, 'add-metric', 'exit 0') - - # First run "install" to make sure all hooks are set up. - state = self._simulate_event(EventSpec(InstallEvent, 'install', charm_config=charm_config)) - self.assertEqual(state['observed_event_types'], [InstallEvent]) - - state = self._simulate_event(EventSpec(ConfigChangedEvent, 'config-changed', - charm_config=charm_config)) - self.assertEqual(state['observed_event_types'], [ConfigChangedEvent]) - - # Re-emit should not pick the deferred config-changed because - # collect-metrics runs in a restricted context. - state = self._simulate_event(EventSpec(CollectMetricsEvent, 'collect-metrics', - charm_config=charm_config)) - self.assertEqual(state['observed_event_types'], [CollectMetricsEvent]) - - def test_multiple_events_handled(self): - self._prepare_actions() - - charm_config = base64.b64encode(pickle.dumps({ - 'STATE_FILE': self._state_file, - })) - actions_charm_config = base64.b64encode(pickle.dumps({ - 'STATE_FILE': self._state_file, - 'USE_ACTIONS': True, - })) - - fake_script(self, 'action-get', "echo '{}'") - - # Sample events with a different amount of dashes used - # and with endpoints from different sections of metadata.yaml - events_under_test = [( - EventSpec(InstallEvent, 'install', - charm_config=charm_config), - {}, - ), ( - EventSpec(StartEvent, 'start', - charm_config=charm_config), - {}, - ), ( - EventSpec(UpdateStatusEvent, 'update_status', - charm_config=charm_config), - {}, - ), ( - EventSpec(LeaderSettingsChangedEvent, 'leader_settings_changed', - charm_config=charm_config), - {}, - ), ( - EventSpec(RelationJoinedEvent, 'db_relation_joined', - relation_id=1, - remote_app='remote', remote_unit='remote/0', - charm_config=charm_config), - {'relation_name': 'db', - 'relation_id': 1, - 'app_name': 'remote', - 'unit_name': 'remote/0'}, - ), ( - EventSpec(RelationChangedEvent, 'mon_relation_changed', - relation_id=2, - remote_app='remote', remote_unit='remote/0', - charm_config=charm_config), - {'relation_name': 'mon', - 'relation_id': 2, - 'app_name': 'remote', - 'unit_name': 'remote/0'}, - ), ( - EventSpec(RelationChangedEvent, 'mon_relation_changed', - relation_id=2, - remote_app='remote', remote_unit=None, - charm_config=charm_config), - {'relation_name': 'mon', - 'relation_id': 2, - 'app_name': 'remote', - 'unit_name': None}, - ), ( - EventSpec(RelationDepartedEvent, 'mon_relation_departed', - relation_id=2, - remote_app='remote', remote_unit='remote/0', - charm_config=charm_config), - {'relation_name': 'mon', - 'relation_id': 2, - 'app_name': 'remote', - 'unit_name': 'remote/0'}, - ), ( - EventSpec(RelationBrokenEvent, 'ha_relation_broken', - relation_id=3, - charm_config=charm_config), - {'relation_name': 'ha', - 'relation_id': 3}, - ), ( - # Events without a remote app specified (for Juju < 2.7). - EventSpec(RelationJoinedEvent, 'db_relation_joined', - relation_id=1, - remote_unit='remote/0', - charm_config=charm_config), - {'relation_name': 'db', - 'relation_id': 1, - 'app_name': 'remote', - 'unit_name': 'remote/0'}, - ), ( - EventSpec(RelationChangedEvent, 'mon_relation_changed', - relation_id=2, - remote_unit='remote/0', - charm_config=charm_config), - {'relation_name': 'mon', - 'relation_id': 2, - 'app_name': 'remote', - 'unit_name': 'remote/0'}, - ), ( - EventSpec(RelationDepartedEvent, 'mon_relation_departed', - relation_id=2, - remote_unit='remote/0', - charm_config=charm_config), - {'relation_name': 'mon', - 'relation_id': 2, - 'app_name': 'remote', - 'unit_name': 'remote/0'}, - ), ( - EventSpec(ActionEvent, 'start_action', - env_var='JUJU_ACTION_NAME', - charm_config=actions_charm_config), - {}, - ), ( - EventSpec(ActionEvent, 'foo_bar_action', - env_var='JUJU_ACTION_NAME', - charm_config=actions_charm_config), - {}, - )] - - logger.debug('Expected events %s', events_under_test) - - # First run "install" to make sure all hooks are set up. - self._simulate_event(EventSpec(InstallEvent, 'install', charm_config=charm_config)) - - # Simulate hook executions for every event. - for event_spec, expected_event_data in events_under_test: - state = self._simulate_event(event_spec) - - state_key = 'on_' + event_spec.event_name - handled_events = state.get(state_key, []) - - # Make sure that a handler for that event was called once. - self.assertEqual(len(handled_events), 1) - # Make sure the event handled by the Charm has the right type. - handled_event_type = handled_events[0] - self.assertEqual(handled_event_type, event_spec.event_type) - - self.assertEqual(state['observed_event_types'], [event_spec.event_type]) - - if event_spec.event_name in expected_event_data: - self.assertEqual(state[event_spec.event_name + '_data'], - expected_event_data[event_spec.event_name]) - - def test_event_not_implemented(self): - """Make sure events without implementation do not cause non-zero exit. - """ - charm_config = base64.b64encode(pickle.dumps({ - 'STATE_FILE': self._state_file, - })) - - # Simulate a scenario where there is a symlink for an event that - # a charm does not know how to handle. - hook_path = self.JUJU_CHARM_DIR / 'hooks/not-implemented-event' - # This will be cleared up in tearDown. - hook_path.symlink_to('install') - - try: - self._simulate_event(EventSpec(HookEvent, 'not-implemented-event', - charm_config=charm_config)) - except subprocess.CalledProcessError: - self.fail('Event simulation for an unsupported event' - ' results in a non-zero exit code returned') - - def test_collect_metrics(self): - indicator_file = self.JUJU_CHARM_DIR / 'indicator' - charm_config = base64.b64encode(pickle.dumps({ - 'STATE_FILE': self._state_file, - 'INDICATOR_FILE': indicator_file - })) - fake_script(self, 'add-metric', 'exit 0') - fake_script(self, 'juju-log', 'exit 0') - self._simulate_event(EventSpec(InstallEvent, 'install', charm_config=charm_config)) - # Clear the calls during 'install' - fake_script_calls(self, clear=True) - self._simulate_event(EventSpec(CollectMetricsEvent, 'collect_metrics', - charm_config=charm_config)) - self.assertEqual( - fake_script_calls(self), - [['juju-log', '--log-level', 'DEBUG', 'Emitting Juju event collect_metrics'], - ['add-metric', '--labels', 'bar=4.2', 'foo=42']]) - - def test_logger(self): - charm_config = base64.b64encode(pickle.dumps({ - 'STATE_FILE': self._state_file, - 'USE_LOG_ACTIONS': True, - })) - fake_script(self, 'action-get', "echo '{}'") - actions_yaml = self.JUJU_CHARM_DIR / 'actions.yaml' - actions_yaml.write_text( - ''' -log_critical: {} -log_error: {} -log_warning: {} -log_info: {} -log_debug: {} - ''') - - test_cases = [( - EventSpec(ActionEvent, 'log_critical_action', env_var='JUJU_ACTION_NAME', - charm_config=charm_config), - ['juju-log', '--log-level', 'CRITICAL', 'super critical'], - ), ( - EventSpec(ActionEvent, 'log_error_action', - env_var='JUJU_ACTION_NAME', - charm_config=charm_config), - ['juju-log', '--log-level', 'ERROR', 'grave error'], - ), ( - EventSpec(ActionEvent, 'log_warning_action', - env_var='JUJU_ACTION_NAME', - charm_config=charm_config), - ['juju-log', '--log-level', 'WARNING', 'wise warning'], - ), ( - EventSpec(ActionEvent, 'log_info_action', - env_var='JUJU_ACTION_NAME', - charm_config=charm_config), - ['juju-log', '--log-level', 'INFO', 'useful info'], - )] - - # Set up action symlinks. - self._simulate_event(EventSpec(InstallEvent, 'install', - charm_config=charm_config)) - - for event_spec, calls in test_cases: - self._simulate_event(event_spec) - self.assertIn(calls, fake_script_calls(self, clear=True)) - - -class TestMainWithNoDispatch(TestMain, unittest.TestCase): - def _setup_entry_point(self, directory, entry_point): - path = directory / entry_point - path.symlink_to(self.charm_exec_path) - - def _call_event(self, rel_path, env): - event_file = self.JUJU_CHARM_DIR / rel_path - # Note that sys.executable is used to make sure we are using the same - # interpreter for the child process to support virtual environments. - subprocess.run( - [sys.executable, str(event_file)], - check=True, env=env, cwd=str(self.JUJU_CHARM_DIR)) - - def test_setup_event_links(self): - """Test auto-creation of symlinks caused by initial events. - """ - all_event_hooks = ['hooks/' + e.replace("_", "-") - for e in self.charm_module.Charm.on.events().keys()] - charm_config = base64.b64encode(pickle.dumps({ - 'STATE_FILE': self._state_file, - })) - initial_events = { - EventSpec(InstallEvent, 'install', charm_config=charm_config), - EventSpec(StorageAttachedEvent, 'disks-storage-attached', charm_config=charm_config), - EventSpec(StartEvent, 'start', charm_config=charm_config), - EventSpec(UpgradeCharmEvent, 'upgrade-charm', charm_config=charm_config), - } - - def _assess_event_links(event_spec): - self.assertTrue(self.hooks_dir / event_spec.event_name in self.hooks_dir.iterdir()) - for event_hook in all_event_hooks: - self.assertTrue((self.JUJU_CHARM_DIR / event_hook).exists(), - 'Missing hook: ' + event_hook) - self.assertEqual(os.readlink(str(self.JUJU_CHARM_DIR / event_hook)), - self.charm_exec_path) - - for initial_event in initial_events: - self._setup_charm_dir() - - self._simulate_event(initial_event) - _assess_event_links(initial_event) - # Make sure it is idempotent. - self._simulate_event(initial_event) - _assess_event_links(initial_event) - - def test_setup_action_links(self): - charm_config = base64.b64encode(pickle.dumps({ - 'STATE_FILE': self._state_file, - })) - actions_yaml = self.JUJU_CHARM_DIR / 'actions.yaml' - actions_yaml.write_text('test: {}') - self._simulate_event(EventSpec(InstallEvent, 'install', charm_config=charm_config)) - action_hook = self.JUJU_CHARM_DIR / 'actions' / 'test' - self.assertTrue(action_hook.exists()) - - -class TestMainWithDispatch(TestMain, unittest.TestCase): - def _setup_entry_point(self, directory, entry_point): - path = self.JUJU_CHARM_DIR / 'dispatch' - if not path.exists(): - path.symlink_to('src/charm.py') - - def _call_event(self, rel_path, env): - env["JUJU_DISPATCH_PATH"] = str(rel_path) - dispatch = self.JUJU_CHARM_DIR / 'dispatch' - subprocess.run( - [sys.executable, str(dispatch)], - stdout=self.stdout, - stderr=self.stderr, - check=True, env=env, cwd=str(self.JUJU_CHARM_DIR)) - - def test_setup_event_links(self): - """Test auto-creation of symlinks caused by initial events does _not_ happen when using dispatch. - """ - all_event_hooks = ['hooks/' + e.replace("_", "-") - for e in self.charm_module.Charm.on.events().keys()] - charm_config = base64.b64encode(pickle.dumps({ - 'STATE_FILE': self._state_file, - })) - initial_events = { - EventSpec(InstallEvent, 'install', charm_config=charm_config), - EventSpec(StorageAttachedEvent, 'disks-storage-attached', charm_config=charm_config), - EventSpec(StartEvent, 'start', charm_config=charm_config), - EventSpec(UpgradeCharmEvent, 'upgrade-charm', charm_config=charm_config), - } - - def _assess_event_links(event_spec): - self.assertNotIn(self.hooks_dir / event_spec.event_name, self.hooks_dir.iterdir()) - for event_hook in all_event_hooks: - self.assertFalse((self.JUJU_CHARM_DIR / event_hook).exists(), - 'Spurious hook: ' + event_hook) - - for initial_event in initial_events: - self._setup_charm_dir() - - self._simulate_event(initial_event) - _assess_event_links(initial_event) - - def test_hook_and_dispatch(self): - charm_config = base64.b64encode(pickle.dumps({ - 'STATE_FILE': self._state_file, - })) - - self.fake_script_path = self.hooks_dir - fake_script(self, 'install', 'exit 0') - state = self._simulate_event(EventSpec(InstallEvent, 'install', charm_config=charm_config)) - - # the script was called, *and*, the .on. was called - self.assertEqual(fake_script_calls(self), [['install', '']]) - self.assertEqual(state['observed_event_types'], [InstallEvent]) - - def test_hook_and_dispatch_with_failing_hook(self): - self.stdout = self.stderr = tempfile.TemporaryFile() - self.addCleanup(self.stdout.close) - - charm_config = base64.b64encode(pickle.dumps({ - 'STATE_FILE': self._state_file, - })) - - old_path = self.fake_script_path - self.fake_script_path = self.hooks_dir - fake_script(self, 'install', 'exit 42') - event = EventSpec(InstallEvent, 'install', charm_config=charm_config) - with self.assertRaises(subprocess.CalledProcessError): - self._simulate_event(event) - self.fake_script_path = old_path - - self.stdout.seek(0) - self.assertEqual(self.stdout.read(), b'') - calls = fake_script_calls(self) - self.assertEqual(len(calls), 1, 'unexpect call result: {}'.format(calls)) - self.assertEqual(len(calls[0]), 4, 'unexpect call result: {}'.format(calls[0])) - self.assertEqual( - calls[0][:3], - ['juju-log', '--log-level', 'WARNING'] - ) - self.assertRegex(calls[0][3], r'hook /\S+/install exited with status 42') - - def test_hook_and_dispatch_but_hook_is_dispatch(self): - charm_config = base64.b64encode(pickle.dumps({ - 'STATE_FILE': self._state_file, - })) - event = EventSpec(InstallEvent, 'install', charm_config=charm_config) - hook_path = self.hooks_dir / 'install' - for ((rel, ind), path) in { - # relative and indirect - (True, True): Path('../dispatch'), - # relative and direct - (True, False): Path(self.charm_exec_path), - # absolute and direct - (False, False): (self.hooks_dir / self.charm_exec_path).resolve(), - # absolute and indirect - (False, True): self.JUJU_CHARM_DIR / 'dispatch', - }.items(): - with self.subTest(path=path, rel=rel, ind=ind): - # sanity check - self.assertEqual(path.is_absolute(), not rel) - self.assertEqual(path.name == 'dispatch', ind) - try: - hook_path.symlink_to(path) - - state = self._simulate_event(event) - - # the .on. was only called once - self.assertEqual(state['observed_event_types'], [InstallEvent]) - self.assertEqual(state['on_install'], [InstallEvent]) - finally: - hook_path.unlink() - - -# TODO: this does not work -# class TestMainWithDispatchAsScript(TestMainWithDispatch): -# """Here dispatch is a script that execs the charm.py instead of a symlink. -# """ -# def _setup_entry_point(self, directory, entry_point): -# path = self.JUJU_CHARM_DIR / 'dispatch' -# if not path.exists(): -# path.write_text('#!/bin/sh\nexec "{}" "{}"\n'.format( -# sys.executable, -# self.JUJU_CHARM_DIR / 'src/charm.py')) -# path.chmod(0o755) - -# def _call_event(self, rel_path, env): -# env["JUJU_DISPATCH_PATH"] = str(rel_path) -# dispatch = self.JUJU_CHARM_DIR / 'dispatch' -# subprocess.check_call([str(dispatch)], -# env=env, cwd=str(self.JUJU_CHARM_DIR)) - - -if __name__ == "__main__": - unittest.main() diff --git a/magma/squid_cnf/charms/squid/mod/operator/test/test_model.py b/magma/squid_cnf/charms/squid/mod/operator/test/test_model.py deleted file mode 100755 index 660176a4556a0d5f7e7ceeb9dbb045d84f727228..0000000000000000000000000000000000000000 --- a/magma/squid_cnf/charms/squid/mod/operator/test/test_model.py +++ /dev/null @@ -1,1288 +0,0 @@ -#!/usr/bin/python3 -# Copyright 2019 Canonical Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -import pathlib -import unittest -import json -import ipaddress -from collections import OrderedDict - -import ops.model -import ops.charm -from ops.charm import RelationMeta - -from test.test_helpers import fake_script, fake_script_calls - - -class TestModel(unittest.TestCase): - - def setUp(self): - def restore_env(env): - os.environ.clear() - os.environ.update(env) - self.addCleanup(restore_env, os.environ.copy()) - - os.environ['JUJU_UNIT_NAME'] = 'myapp/0' - - self.backend = ops.model.ModelBackend() - meta = ops.charm.CharmMeta() - meta.relations = { - 'db0': RelationMeta('provides', 'db0', {'interface': 'db0', 'scope': 'global'}), - 'db1': RelationMeta('requires', 'db1', {'interface': 'db1', 'scope': 'global'}), - 'db2': RelationMeta('peers', 'db2', {'interface': 'db2', 'scope': 'global'}), - } - self.model = ops.model.Model('myapp/0', meta, self.backend) - fake_script(self, 'relation-ids', """([ "$1" = db0 ] && echo '["db0:4"]') || echo '[]'""") - - def test_model(self): - self.assertIs(self.model.app, self.model.unit.app) - - def test_relations_keys(self): - fake_script(self, 'relation-ids', - """[ "$1" = db2 ] && echo '["db2:5", "db2:6"]' || echo '[]'""") - fake_script(self, 'relation-list', - """ -case "$2" in - 5) - echo '["remoteapp1/0", "remoteapp1/1"]' - ;; - 6) - echo '["remoteapp2/0"]' - ;; - *) - exit 2 - ;; -esac -""") - - for relation in self.model.relations['db2']: - self.assertIn(self.model.unit, relation.data) - unit_from_rel = next(filter(lambda u: u.name == 'myapp/0', relation.data.keys())) - self.assertIs(self.model.unit, unit_from_rel) - - self.assertEqual(fake_script_calls(self), [ - ['relation-ids', 'db2', '--format=json'], - ['relation-list', '-r', '5', '--format=json'], - ['relation-list', '-r', '6', '--format=json'] - ]) - - def test_get_relation(self): - err_msg = 'ERROR invalid value "$2" for option -r: relation not found' - - fake_script(self, 'relation-ids', ''' - case "$1" in - db1) - echo '["db1:4"]' - ;; - db2) - echo '["db2:5", "db2:6"]' - ;; - *) - echo '[]' - ;; - esac - ''') - fake_script(self, 'relation-list', ''' - if [ "$2" = 4 ]; then - echo '["remoteapp1/0"]' - else - echo {} >&2 - exit 2 - fi - '''.format(err_msg)) - fake_script(self, 'relation-get', - """echo {} >&2 ; exit 2""".format(err_msg)) - - with self.assertRaises(ops.model.ModelError): - self.model.get_relation('db1', 'db1:4') - db1_4 = self.model.get_relation('db1', 4) - self.assertIsInstance(db1_4, ops.model.Relation) - dead_rel = self.model.get_relation('db1', 7) - self.assertIsInstance(dead_rel, ops.model.Relation) - self.assertEqual(set(dead_rel.data.keys()), {self.model.unit, self.model.unit.app}) - self.assertEqual(dead_rel.data[self.model.unit], {}) - self.assertIsNone(self.model.get_relation('db0')) - self.assertIs(self.model.get_relation('db1'), db1_4) - with self.assertRaises(ops.model.TooManyRelatedAppsError): - self.model.get_relation('db2') - - self.assertEqual(fake_script_calls(self), [ - ['relation-ids', 'db1', '--format=json'], - ['relation-list', '-r', '4', '--format=json'], - ['relation-list', '-r', '7', '--format=json'], - ['relation-get', '-r', '7', '-', 'myapp/0', '--app=False', '--format=json'], - ['relation-ids', 'db0', '--format=json'], - ['relation-ids', 'db2', '--format=json'], - ['relation-list', '-r', '5', '--format=json'], - ['relation-list', '-r', '6', '--format=json'] - ]) - - def test_peer_relation_app(self): - meta = ops.charm.CharmMeta() - meta.relations = {'dbpeer': RelationMeta('peers', 'dbpeer', - {'interface': 'dbpeer', 'scope': 'global'})} - self.model = ops.model.Model('myapp/0', meta, self.backend) - - err_msg = 'ERROR invalid value "$2" for option -r: relation not found' - fake_script(self, 'relation-ids', - '''([ "$1" = dbpeer ] && echo '["dbpeer:0"]') || echo "[]"''') - fake_script(self, 'relation-list', - '''([ "$2" = 0 ] && echo "[]") || (echo {} >&2 ; exit 2)'''.format(err_msg)) - - db1_4 = self.model.get_relation('dbpeer') - self.assertIs(db1_4.app, self.model.app) - - def test_remote_units_is_our(self): - fake_script(self, 'relation-ids', - """[ "$1" = db1 ] && echo '["db1:4"]' || echo '[]'""") - fake_script(self, 'relation-list', - """[ "$2" = 4 ] && echo '["remoteapp1/0", "remoteapp1/1"]' || exit 2""") - - for u in self.model.get_relation('db1').units: - self.assertFalse(u._is_our_unit) - self.assertFalse(u.app._is_our_app) - - self.assertEqual(fake_script_calls(self), [ - ['relation-ids', 'db1', '--format=json'], - ['relation-list', '-r', '4', '--format=json'] - ]) - - def test_our_unit_is_our(self): - self.assertTrue(self.model.unit._is_our_unit) - self.assertTrue(self.model.unit.app._is_our_app) - - def test_unit_relation_data(self): - fake_script(self, 'relation-ids', """[ "$1" = db1 ] && echo '["db1:4"]' || echo '[]'""") - fake_script(self, 'relation-list', """[ "$2" = 4 ] && echo '["remoteapp1/0"]' || exit 2""") - fake_script(self, 'relation-get', """ -if [ "$2" = 4 ] && [ "$4" = "remoteapp1/0" ]; then - echo '{"host": "remoteapp1-0"}' -else - exit 2 -fi -""") - - random_unit = self.model._cache.get(ops.model.Unit, 'randomunit/0') - with self.assertRaises(KeyError): - self.model.get_relation('db1').data[random_unit] - remoteapp1_0 = next(filter(lambda u: u.name == 'remoteapp1/0', - self.model.get_relation('db1').units)) - self.assertEqual(self.model.get_relation('db1').data[remoteapp1_0], - {'host': 'remoteapp1-0'}) - - self.assertEqual(fake_script_calls(self), [ - ['relation-ids', 'db1', '--format=json'], - ['relation-list', '-r', '4', '--format=json'], - ['relation-get', '-r', '4', '-', 'remoteapp1/0', '--app=False', '--format=json'] - ]) - - def test_remote_app_relation_data(self): - fake_script(self, 'relation-ids', """[ "$1" = db1 ] && echo '["db1:4"]' || echo '[]'""") - fake_script(self, 'relation-list', ''' - if [ "$2" = 4 ]; then - echo '["remoteapp1/0", "remoteapp1/1"]' - else - exit 2 - fi - ''') - fake_script(self, 'relation-get', ''' - if [ "$2" = 4 ] && [ "$4" = remoteapp1 ]; then - echo '{"secret": "cafedeadbeef"}' - else - exit 2 - fi - ''') - - # Try to get relation data for an invalid remote application. - random_app = self.model._cache.get(ops.model.Application, 'randomapp') - with self.assertRaises(KeyError): - self.model.get_relation('db1').data[random_app] - - remoteapp1 = self.model.get_relation('db1').app - self.assertEqual(self.model.get_relation('db1').data[remoteapp1], - {'secret': 'cafedeadbeef'}) - - self.assertEqual(fake_script_calls(self), [ - ['relation-ids', 'db1', '--format=json'], - ['relation-list', '-r', '4', '--format=json'], - ['relation-get', '-r', '4', '-', 'remoteapp1', '--app=True', '--format=json'], - ]) - - def test_relation_data_modify_remote(self): - fake_script(self, 'relation-ids', """[ "$1" = db1 ] && echo '["db1:4"]' || echo '[]'""") - fake_script(self, 'relation-list', """[ "$2" = 4 ] && echo '["remoteapp1/0"]' || exit 2""") - fake_script(self, 'relation-get', """ -if [ "$2" = 4 ] && [ "$4" = "remoteapp1/0" ]; then - echo '{"host": "remoteapp1-0"}' -else - exit 2 -fi -""") - - rel_db1 = self.model.get_relation('db1') - remoteapp1_0 = next(filter(lambda u: u.name == 'remoteapp1/0', - self.model.get_relation('db1').units)) - # Force memory cache to be loaded. - self.assertIn('host', rel_db1.data[remoteapp1_0]) - with self.assertRaises(ops.model.RelationDataError): - rel_db1.data[remoteapp1_0]['foo'] = 'bar' - self.assertNotIn('foo', rel_db1.data[remoteapp1_0]) - - self.assertEqual(fake_script_calls(self), [ - ['relation-ids', 'db1', '--format=json'], - ['relation-list', '-r', '4', '--format=json'], - ['relation-get', '-r', '4', '-', 'remoteapp1/0', '--app=False', '--format=json'] - ]) - - def test_relation_data_modify_our(self): - fake_script(self, 'relation-ids', """[ "$1" = db1 ] && echo '["db1:4"]' || echo '[]'""") - fake_script(self, 'relation-list', """[ "$2" = 4 ] && echo '["remoteapp1/0"]' || exit 2""") - fake_script(self, 'relation-set', '''[ "$2" = 4 ] && exit 0 || exit 2''') - fake_script(self, 'relation-get', ''' - if [ "$2" = 4 ] && [ "$4" = "myapp/0" ]; then - echo '{"host": "bar"}' - else - exit 2 - fi - ''') - - rel_db1 = self.model.get_relation('db1') - # Force memory cache to be loaded. - self.assertIn('host', rel_db1.data[self.model.unit]) - rel_db1.data[self.model.unit]['host'] = 'bar' - self.assertEqual(rel_db1.data[self.model.unit]['host'], 'bar') - - self.assertEqual(fake_script_calls(self), [ - ['relation-ids', 'db1', '--format=json'], - ['relation-list', '-r', '4', '--format=json'], - ['relation-get', '-r', '4', '-', 'myapp/0', '--app=False', '--format=json'], - ['relation-set', '-r', '4', 'host=bar', '--app=False'] - ]) - - def test_app_relation_data_modify_local_as_leader(self): - fake_script(self, 'relation-ids', """[ "$1" = db1 ] && echo '["db1:4"]' || echo '[]'""") - fake_script(self, 'relation-list', ''' - if [ "$2" = 4 ]; then - echo '["remoteapp1/0", "remoteapp1/1"]' - else - exit 2 - fi - ''') - fake_script(self, 'relation-get', ''' - if [ "$2" = 4 ] && [ "$4" = myapp ]; then - echo '{"password": "deadbeefcafe"}' - else - exit 2 - fi - ''') - fake_script(self, 'relation-set', """[ "$2" = 4 ] && exit 0 || exit 2""") - fake_script(self, 'is-leader', 'echo true') - - local_app = self.model.unit.app - - rel_db1 = self.model.get_relation('db1') - self.assertEqual(rel_db1.data[local_app], {'password': 'deadbeefcafe'}) - - rel_db1.data[local_app]['password'] = 'foo' - - self.assertEqual(rel_db1.data[local_app]['password'], 'foo') - - self.assertEqual(fake_script_calls(self), [ - ['relation-ids', 'db1', '--format=json'], - ['relation-list', '-r', '4', '--format=json'], - ['relation-get', '-r', '4', '-', 'myapp', '--app=True', '--format=json'], - ['is-leader', '--format=json'], - ['relation-set', '-r', '4', 'password=foo', '--app=True'], - ]) - - def test_app_relation_data_modify_local_as_minion(self): - fake_script(self, 'relation-ids', """[ "$1" = db1 ] && echo '["db1:4"]' || echo '[]'""") - fake_script(self, 'relation-list', ''' - if [ "$2" = 4 ]; then - echo '["remoteapp1/0", "remoteapp1/1"]' - else - exit 2 - fi - ''') - fake_script(self, 'relation-get', ''' - if [ "$2" = 4 ] && [ "$4" = myapp ]; then - echo '{"password": "deadbeefcafe"}' - else - exit 2 - fi - ''') - fake_script(self, 'is-leader', 'echo false') - - local_app = self.model.unit.app - - rel_db1 = self.model.get_relation('db1') - self.assertEqual(rel_db1.data[local_app], {'password': 'deadbeefcafe'}) - - with self.assertRaises(ops.model.RelationDataError): - rel_db1.data[local_app]['password'] = 'foobar' - - self.assertEqual(fake_script_calls(self), [ - ['relation-ids', 'db1', '--format=json'], - ['relation-list', '-r', '4', '--format=json'], - ['relation-get', '-r', '4', '-', 'myapp', '--app=True', '--format=json'], - ['is-leader', '--format=json'], - ]) - - def test_relation_data_del_key(self): - fake_script(self, 'relation-ids', """[ "$1" = db1 ] && echo '["db1:4"]' || echo '[]'""") - fake_script(self, 'relation-list', """[ "$2" = 4 ] && echo '["remoteapp1/0"]' || exit 2""") - fake_script(self, 'relation-set', '''[ "$2" = 4 ] && exit 0 || exit 2''') - fake_script(self, 'relation-get', ''' - if [ "$2" = 4 ] && [ "$4" = "myapp/0" ]; then - echo '{"host": "bar"}' - else - exit 2 - fi - ''') - - rel_db1 = self.model.get_relation('db1') - # Force memory cache to be loaded. - self.assertIn('host', rel_db1.data[self.model.unit]) - del rel_db1.data[self.model.unit]['host'] - fake_script(self, 'relation-get', ''' - if [ "$2" = 4 ] && [ "$4" = "myapp/0" ]; then - echo '{}' - else - exit 2 - fi - ''') - self.assertNotIn('host', rel_db1.data[self.model.unit]) - - self.assertEqual(fake_script_calls(self), [ - ['relation-ids', 'db1', '--format=json'], - ['relation-list', '-r', '4', '--format=json'], - ['relation-get', '-r', '4', '-', 'myapp/0', '--app=False', '--format=json'], - ['relation-set', '-r', '4', 'host=', '--app=False'] - ]) - - def test_relation_set_fail(self): - fake_script(self, 'relation-ids', """[ "$1" = db2 ] && echo '["db2:5"]' || echo '[]'""") - fake_script(self, 'relation-list', - """[ "$2" = 5 ] && echo '["remoteapp1/0"]' || exit 2""") - fake_script(self, 'relation-get', ''' - if [ "$2" = 5 ] && [ "$4" = "myapp/0" ]; then - echo '{"host": "myapp-0"}' - else - exit 2 - fi - ''') - fake_script(self, 'relation-set', 'exit 2') - - rel_db2 = self.model.relations['db2'][0] - # Force memory cache to be loaded. - self.assertIn('host', rel_db2.data[self.model.unit]) - with self.assertRaises(ops.model.ModelError): - rel_db2.data[self.model.unit]['host'] = 'bar' - self.assertEqual(rel_db2.data[self.model.unit]['host'], 'myapp-0') - with self.assertRaises(ops.model.ModelError): - del rel_db2.data[self.model.unit]['host'] - self.assertIn('host', rel_db2.data[self.model.unit]) - - self.assertEqual(fake_script_calls(self), [ - ['relation-ids', 'db2', '--format=json'], - ['relation-list', '-r', '5', '--format=json'], - ['relation-get', '-r', '5', '-', 'myapp/0', '--app=False', '--format=json'], - ['relation-set', '-r', '5', 'host=bar', '--app=False'], - ['relation-set', '-r', '5', 'host=', '--app=False'] - ]) - - def test_relation_get_set_is_app_arg(self): - self.backend = ops.model.ModelBackend() - - # No is_app provided. - with self.assertRaises(TypeError): - self.backend.relation_set(1, 'fookey', 'barval') - - with self.assertRaises(TypeError): - self.backend.relation_get(1, 'fooentity') - - # Invalid types for is_app. - for is_app_v in [None, 1, 2.0, 'a', b'beef']: - with self.assertRaises(TypeError): - self.backend.relation_set(1, 'fookey', 'barval', is_app=is_app_v) - - with self.assertRaises(TypeError): - self.backend.relation_get(1, 'fooentity', is_app=is_app_v) - - def test_relation_data_type_check(self): - fake_script(self, 'relation-ids', """[ "$1" = db1 ] && echo '["db1:4"]' || echo '[]'""") - fake_script(self, 'relation-list', - """[ "$2" = 4 ] && echo '["remoteapp1/0"]' || exit 2""") - fake_script(self, 'relation-get', ''' - if [ "$2" = 4 ] && [ "$4" = "myapp/0" ]; then - echo '{"host": "myapp-0"}' - else - exit 2 - fi - ''') - - rel_db1 = self.model.get_relation('db1') - with self.assertRaises(ops.model.RelationDataError): - rel_db1.data[self.model.unit]['foo'] = 1 - with self.assertRaises(ops.model.RelationDataError): - rel_db1.data[self.model.unit]['foo'] = {'foo': 'bar'} - with self.assertRaises(ops.model.RelationDataError): - rel_db1.data[self.model.unit]['foo'] = None - - self.assertEqual(fake_script_calls(self), [ - ['relation-ids', 'db1', '--format=json'], - ['relation-list', '-r', '4', '--format=json'] - ]) - - def test_config(self): - fake_script(self, 'config-get', """echo '{"foo":"foo","bar":1,"qux":true}'""") - self.assertEqual(self.model.config, { - 'foo': 'foo', - 'bar': 1, - 'qux': True, - }) - with self.assertRaises(TypeError): - # Confirm that we cannot modify config values. - self.model.config['foo'] = 'bar' - - self.assertEqual(fake_script_calls(self), [['config-get', '--format=json']]) - - def test_is_leader(self): - def check_remote_units(): - fake_script(self, 'relation-ids', - """[ "$1" = db1 ] && echo '["db1:4"]' || echo '[]'""") - - fake_script(self, 'relation-list', - """[ "$2" = 4 ] && echo '["remoteapp1/0", "remoteapp1/1"]' || exit 2""") - - # Cannot determine leadership for remote units. - for u in self.model.get_relation('db1').units: - with self.assertRaises(RuntimeError): - u.is_leader() - - fake_script(self, 'is-leader', 'echo true') - self.assertTrue(self.model.unit.is_leader()) - - check_remote_units() - - # Create a new model and backend to drop a cached is-leader output. - self.backend = ops.model.ModelBackend() - meta = ops.charm.CharmMeta() - meta.relations = { - 'db0': RelationMeta('provides', 'db0', {'interface': 'db0', 'scope': 'global'}), - 'db1': RelationMeta('requires', 'db1', {'interface': 'db1', 'scope': 'global'}), - 'db2': RelationMeta('peers', 'db2', {'interface': 'db2', 'scope': 'global'}), - } - self.model = ops.model.Model('myapp/0', meta, self.backend) - - fake_script(self, 'is-leader', 'echo false') - self.assertFalse(self.model.unit.is_leader()) - - check_remote_units() - - self.assertEqual(fake_script_calls(self), [ - ['is-leader', '--format=json'], - ['relation-ids', 'db1', '--format=json'], - ['relation-list', '-r', '4', '--format=json'], - ['is-leader', '--format=json'], - ['relation-ids', 'db1', '--format=json'], - ['relation-list', '-r', '4', '--format=json'], - ]) - - def test_is_leader_refresh(self): - fake_script(self, 'is-leader', 'echo false') - self.assertFalse(self.model.unit.is_leader()) - - # Change the leadership status and force a recheck. - fake_script(self, 'is-leader', 'echo true') - self.backend._leader_check_time = None - self.assertTrue(self.model.unit.is_leader()) - - # Force a recheck without changing the leadership status. - fake_script(self, 'is-leader', 'echo true') - self.backend._leader_check_time = None - self.assertTrue(self.model.unit.is_leader()) - - def test_workload_version(self): - fake_script(self, 'application-version-set', 'exit 0') - self.model.unit.set_workload_version('1.2.3') - self.assertEqual(fake_script_calls(self), [['application-version-set', '--', '1.2.3']]) - - def test_workload_version_invalid(self): - fake_script(self, 'application-version-set', 'exit 0') - with self.assertRaises(TypeError) as cm: - self.model.unit.set_workload_version(5) - self.assertEqual(str(cm.exception), "workload version must be a str, not int: 5") - self.assertEqual(fake_script_calls(self), []) - - def test_resources(self): - meta = ops.charm.CharmMeta() - meta.resources = {'foo': None, 'bar': None} - model = ops.model.Model('myapp/0', meta, self.backend) - - with self.assertRaises(RuntimeError): - model.resources.fetch('qux') - - fake_script(self, 'resource-get', 'exit 1') - with self.assertRaises(ops.model.ModelError): - model.resources.fetch('foo') - - fake_script(self, 'resource-get', - 'echo /var/lib/juju/agents/unit-test-0/resources/$1/$1.tgz') - self.assertEqual(model.resources.fetch('foo').name, 'foo.tgz') - self.assertEqual(model.resources.fetch('bar').name, 'bar.tgz') - - def test_pod_spec(self): - fake_script(self, 'pod-spec-set', """ - cat $2 > $(dirname $0)/spec.json - [[ -n $4 ]] && cat $4 > $(dirname $0)/k8s_res.json || true - """) - fake_script(self, 'is-leader', 'echo true') - spec_path = self.fake_script_path / 'spec.json' - k8s_res_path = self.fake_script_path / 'k8s_res.json' - - def check_calls(calls): - # There may 1 or 2 calls because of is-leader. - self.assertLessEqual(len(fake_calls), 2) - pod_spec_call = next(filter(lambda c: c[0] == 'pod-spec-set', calls)) - self.assertEqual(pod_spec_call[:2], ['pod-spec-set', '--file']) - - # 8 bytes are used as of python 3.4.0, see Python bug #12015. - # Other characters are from POSIX 3.282 (Portable Filename - # Character Set) a subset of which Python's mkdtemp uses. - self.assertRegex(pod_spec_call[2], '.*/tmp[A-Za-z0-9._-]{8}-pod-spec-set') - - self.model.pod.set_spec({'foo': 'bar'}) - self.assertEqual(spec_path.read_text(), '{"foo": "bar"}') - self.assertFalse(k8s_res_path.exists()) - - fake_calls = fake_script_calls(self, clear=True) - check_calls(fake_calls) - - self.model.pod.set_spec({'bar': 'foo'}, {'qux': 'baz'}) - self.assertEqual(spec_path.read_text(), '{"bar": "foo"}') - self.assertEqual(k8s_res_path.read_text(), '{"qux": "baz"}') - - fake_calls = fake_script_calls(self, clear=True) - check_calls(fake_calls) - - # Create a new model to drop is-leader caching result. - self.backend = ops.model.ModelBackend() - meta = ops.charm.CharmMeta() - self.model = ops.model.Model('myapp/0', meta, self.backend) - fake_script(self, 'is-leader', 'echo false') - with self.assertRaises(ops.model.ModelError): - self.model.pod.set_spec({'foo': 'bar'}) - - def test_base_status_instance_raises(self): - with self.assertRaises(TypeError): - ops.model.StatusBase('test') - - def test_active_message_default(self): - self.assertEqual(ops.model.ActiveStatus().message, '') - - def test_local_set_valid_unit_status(self): - test_cases = [( - ops.model.ActiveStatus('Green'), - lambda: fake_script(self, 'status-set', 'exit 0'), - lambda: self.assertEqual(fake_script_calls(self, True), - [['status-set', '--application=False', 'active', 'Green']]), - ), ( - ops.model.MaintenanceStatus('Yellow'), - lambda: fake_script(self, 'status-set', 'exit 0'), - lambda: self.assertEqual( - fake_script_calls(self, True), - [['status-set', '--application=False', 'maintenance', 'Yellow']]), - ), ( - ops.model.BlockedStatus('Red'), - lambda: fake_script(self, 'status-set', 'exit 0'), - lambda: self.assertEqual(fake_script_calls(self, True), - [['status-set', '--application=False', 'blocked', 'Red']]), - ), ( - ops.model.WaitingStatus('White'), - lambda: fake_script(self, 'status-set', 'exit 0'), - lambda: self.assertEqual(fake_script_calls(self, True), - [['status-set', '--application=False', 'waiting', 'White']]), - )] - - for target_status, setup_tools, check_tool_calls in test_cases: - setup_tools() - - self.model.unit.status = target_status - - self.assertEqual(self.model.unit.status, target_status) - - check_tool_calls() - - def test_local_set_valid_app_status(self): - fake_script(self, 'is-leader', 'echo true') - test_cases = [( - ops.model.ActiveStatus('Green'), - lambda: fake_script(self, 'status-set', 'exit 0'), - lambda: self.assertIn(['status-set', '--application=True', 'active', 'Green'], - fake_script_calls(self, True)), - ), ( - ops.model.MaintenanceStatus('Yellow'), - lambda: fake_script(self, 'status-set', 'exit 0'), - lambda: self.assertIn(['status-set', '--application=True', 'maintenance', 'Yellow'], - fake_script_calls(self, True)), - ), ( - ops.model.BlockedStatus('Red'), - lambda: fake_script(self, 'status-set', 'exit 0'), - lambda: self.assertIn(['status-set', '--application=True', 'blocked', 'Red'], - fake_script_calls(self, True)), - ), ( - ops.model.WaitingStatus('White'), - lambda: fake_script(self, 'status-set', 'exit 0'), - lambda: self.assertIn(['status-set', '--application=True', 'waiting', 'White'], - fake_script_calls(self, True)), - )] - - for target_status, setup_tools, check_tool_calls in test_cases: - setup_tools() - - self.model.app.status = target_status - - self.assertEqual(self.model.app.status, target_status) - - check_tool_calls() - - def test_set_app_status_non_leader_raises(self): - fake_script(self, 'is-leader', 'echo false') - - with self.assertRaises(RuntimeError): - self.model.app.status - - with self.assertRaises(RuntimeError): - self.model.app.status = ops.model.ActiveStatus() - - def test_local_set_invalid_status(self): - fake_script(self, 'status-set', 'exit 1') - fake_script(self, 'is-leader', 'echo true') - - with self.assertRaises(ops.model.ModelError): - self.model.unit.status = ops.model.UnknownStatus() - - self.assertEqual(fake_script_calls(self, True), [ - ['status-set', '--application=False', 'unknown', ''], - ]) - - with self.assertRaises(ops.model.ModelError): - self.model.app.status = ops.model.UnknownStatus() - - # A leadership check is needed for application status. - self.assertEqual(fake_script_calls(self, True), [ - ['is-leader', '--format=json'], - ['status-set', '--application=True', 'unknown', ''], - ]) - - def test_status_set_is_app_not_bool_raises(self): - self.backend = ops.model.ModelBackend() - - for is_app_v in [None, 1, 2.0, 'a', b'beef', object]: - with self.assertRaises(TypeError): - self.backend.status_set(ops.model.ActiveStatus, is_app=is_app_v) - - def test_remote_unit_status(self): - fake_script(self, 'relation-ids', """[ "$1" = db1 ] && echo '["db1:4"]' || echo '[]'""") - fake_script(self, 'relation-list', ''' - if [ "$2" = 4 ]; then - echo '["remoteapp1/0", "remoteapp1/1"]' - else - exit 2 - fi - ''') - - remote_unit = next(filter(lambda u: u.name == 'remoteapp1/0', - self.model.get_relation('db1').units)) - - test_statuses = ( - ops.model.UnknownStatus(), - ops.model.ActiveStatus('Green'), - ops.model.MaintenanceStatus('Yellow'), - ops.model.BlockedStatus('Red'), - ops.model.WaitingStatus('White'), - ) - - for target_status in test_statuses: - with self.assertRaises(RuntimeError): - remote_unit.status = target_status - - def test_remote_app_status(self): - fake_script(self, 'relation-ids', """[ "$1" = db1 ] && echo '["db1:4"]' || echo '[]'""") - fake_script(self, 'relation-list', ''' - if [ "$2" = 4 ]; then - echo '["remoteapp1/0", "remoteapp1/1"]' - else - exit 2 - fi - ''') - - remoteapp1 = self.model.get_relation('db1').app - - # Remote application status is always unknown. - self.assertIsInstance(remoteapp1.status, ops.model.UnknownStatus) - - test_statuses = ( - ops.model.UnknownStatus(), - ops.model.ActiveStatus(), - ops.model.MaintenanceStatus('Upgrading software'), - ops.model.BlockedStatus('Awaiting manual resolution'), - ops.model.WaitingStatus('Awaiting related app updates'), - ) - for target_status in test_statuses: - with self.assertRaises(RuntimeError): - remoteapp1.status = target_status - - self.assertEqual(fake_script_calls(self, clear=True), [ - ['relation-ids', 'db1', '--format=json'], - ['relation-list', '-r', '4', '--format=json'], - ]) - - def test_storage(self): - meta = ops.charm.CharmMeta() - meta.storages = {'disks': None, 'data': None} - self.model = ops.model.Model('myapp/0', meta, self.backend) - - fake_script(self, 'storage-list', ''' - if [ "$1" = disks ]; then - echo '["disks/0", "disks/1"]' - else - echo '[]' - fi - ''') - fake_script(self, 'storage-get', ''' - if [ "$2" = disks/0 ]; then - echo '"/var/srv/disks/0"' - elif [ "$2" = disks/1 ]; then - echo '"/var/srv/disks/1"' - else - exit 2 - fi - ''') - fake_script(self, 'storage-add', '') - - self.assertEqual(len(self.model.storages), 2) - self.assertEqual(self.model.storages.keys(), meta.storages.keys()) - self.assertIn('disks', self.model.storages) - test_cases = { - 0: {'name': 'disks', 'location': pathlib.Path('/var/srv/disks/0')}, - 1: {'name': 'disks', 'location': pathlib.Path('/var/srv/disks/1')}, - } - for storage in self.model.storages['disks']: - self.assertEqual(storage.name, 'disks') - self.assertIn(storage.id, test_cases) - self.assertEqual(storage.name, test_cases[storage.id]['name']) - self.assertEqual(storage.location, test_cases[storage.id]['location']) - - self.assertEqual(fake_script_calls(self, clear=True), [ - ['storage-list', 'disks', '--format=json'], - ['storage-get', '-s', 'disks/0', 'location', '--format=json'], - ['storage-get', '-s', 'disks/1', 'location', '--format=json'], - ]) - - self.assertSequenceEqual(self.model.storages['data'], []) - self.model.storages.request('data', count=3) - self.assertEqual(fake_script_calls(self), [ - ['storage-list', 'data', '--format=json'], - ['storage-add', 'data=3'], - ]) - - # Try to add storage not present in charm metadata. - with self.assertRaises(ops.model.ModelError): - self.model.storages.request('deadbeef') - - # Invalid count parameter types. - for count_v in [None, False, 2.0, 'a', b'beef', object]: - with self.assertRaises(TypeError): - self.model.storages.request('data', count_v) - - -class TestModelBindings(unittest.TestCase): - - def setUp(self): - def restore_env(env): - os.environ.clear() - os.environ.update(env) - self.addCleanup(restore_env, os.environ.copy()) - - os.environ['JUJU_UNIT_NAME'] = 'myapp/0' - - meta = ops.charm.CharmMeta() - meta.relations = { - 'db0': RelationMeta('provides', 'db0', {'interface': 'db0', 'scope': 'global'}), - 'db1': RelationMeta('requires', 'db1', {'interface': 'db1', 'scope': 'global'}), - 'db2': RelationMeta('peers', 'db2', {'interface': 'db2', 'scope': 'global'}), - } - self.backend = ops.model.ModelBackend() - self.model = ops.model.Model('myapp/0', meta, self.backend) - - fake_script(self, 'relation-ids', - """([ "$1" = db0 ] && echo '["db0:4"]') || echo '[]'""") - fake_script(self, 'relation-list', """[ "$2" = 4 ] && echo '["remoteapp1/0"]' || exit 2""") - self.network_get_out = '''{ - "bind-addresses": [ - { - "mac-address": "de:ad:be:ef:ca:fe", - "interface-name": "lo", - "addresses": [ - { - "hostname": "", - "value": "192.0.2.2", - "cidr": "192.0.2.0/24" - }, - { - "hostname": "deadbeef.example", - "value": "dead:beef::1", - "cidr": "dead:beef::/64" - } - ] - }, - { - "mac-address": "", - "interface-name": "tun", - "addresses": [ - { - "hostname": "", - "value": "192.0.3.3", - "cidr": "" - }, - { - "hostname": "", - "value": "2001:db8::3", - "cidr": "" - }, - { - "hostname": "deadbeef.local", - "value": "fe80::1:1", - "cidr": "fe80::/64" - } - ] - } - ], - "egress-subnets": [ - "192.0.2.2/32", - "192.0.3.0/24", - "dead:beef::/64", - "2001:db8::3/128" - ], - "ingress-addresses": [ - "192.0.2.2", - "192.0.3.3", - "dead:beef::1", - "2001:db8::3" - ] -}''' - - def _check_binding_data(self, binding_name, binding): - self.assertEqual(binding.name, binding_name) - self.assertEqual(binding.network.bind_address, ipaddress.ip_address('192.0.2.2')) - self.assertEqual(binding.network.ingress_address, ipaddress.ip_address('192.0.2.2')) - # /32 and /128 CIDRs are valid one-address networks for IPv{4,6}Network types respectively. - self.assertEqual(binding.network.egress_subnets, [ipaddress.ip_network('192.0.2.2/32'), - ipaddress.ip_network('192.0.3.0/24'), - ipaddress.ip_network('dead:beef::/64'), - ipaddress.ip_network('2001:db8::3/128')]) - - for (i, (name, address, subnet)) in enumerate([ - ('lo', '192.0.2.2', '192.0.2.0/24'), - ('lo', 'dead:beef::1', 'dead:beef::/64'), - ('tun', '192.0.3.3', '192.0.3.3/32'), - ('tun', '2001:db8::3', '2001:db8::3/128'), - ('tun', 'fe80::1:1', 'fe80::/64')]): - self.assertEqual(binding.network.interfaces[i].name, name) - self.assertEqual(binding.network.interfaces[i].address, ipaddress.ip_address(address)) - self.assertEqual(binding.network.interfaces[i].subnet, ipaddress.ip_network(subnet)) - - def test_invalid_keys(self): - # Basic validation for passing invalid keys. - for name in (object, 0): - with self.assertRaises(ops.model.ModelError): - self.model.get_binding(name) - - def test_dead_relations(self): - fake_script( - self, - 'network-get', - ''' - if [ "$1" = db0 ] && [ "$2" = --format=json ]; then - echo '{}' - else - echo ERROR invalid value "$2" for option -r: relation not found >&2 - exit 2 - fi - '''.format(self.network_get_out)) - # Validate the behavior for dead relations. - binding = ops.model.Binding('db0', 42, self.model._backend) - self.assertEqual(binding.network.bind_address, ipaddress.ip_address('192.0.2.2')) - self.assertEqual(fake_script_calls(self, clear=True), [ - ['network-get', 'db0', '-r', '42', '--format=json'], - ['network-get', 'db0', '--format=json'], - ]) - - def test_binding_by_relation_name(self): - fake_script(self, 'network-get', - '''[ "$1" = db0 ] && echo '{}' || exit 1'''.format(self.network_get_out)) - binding_name = 'db0' - expected_calls = [['network-get', 'db0', '--format=json']] - - binding = self.model.get_binding(binding_name) - self._check_binding_data(binding_name, binding) - self.assertEqual(fake_script_calls(self, clear=True), expected_calls) - - def test_binding_by_relation(self): - fake_script(self, 'network-get', - '''[ "$1" = db0 ] && echo '{}' || exit 1'''.format(self.network_get_out)) - binding_name = 'db0' - expected_calls = [ - ['relation-ids', 'db0', '--format=json'], - # The two invocations below are due to the get_relation call. - ['relation-list', '-r', '4', '--format=json'], - ['network-get', 'db0', '-r', '4', '--format=json'], - ] - binding = self.model.get_binding(self.model.get_relation(binding_name)) - self._check_binding_data(binding_name, binding) - self.assertEqual(fake_script_calls(self, clear=True), expected_calls) - - -class TestModelBackend(unittest.TestCase): - - def setUp(self): - os.environ['JUJU_UNIT_NAME'] = 'myapp/0' - self.addCleanup(os.environ.pop, 'JUJU_UNIT_NAME') - - self._backend = None - - @property - def backend(self): - if self._backend is None: - self._backend = ops.model.ModelBackend() - return self._backend - - def test_relation_tool_errors(self): - err_msg = 'ERROR invalid value "$2" for option -r: relation not found' - - test_cases = [( - lambda: fake_script(self, 'relation-list', 'echo fooerror >&2 ; exit 1'), - lambda: self.backend.relation_list(3), - ops.model.ModelError, - [['relation-list', '-r', '3', '--format=json']], - ), ( - lambda: fake_script(self, 'relation-list', 'echo {} >&2 ; exit 2'.format(err_msg)), - lambda: self.backend.relation_list(3), - ops.model.RelationNotFoundError, - [['relation-list', '-r', '3', '--format=json']], - ), ( - lambda: fake_script(self, 'relation-set', 'echo fooerror >&2 ; exit 1'), - lambda: self.backend.relation_set(3, 'foo', 'bar', is_app=False), - ops.model.ModelError, - [['relation-set', '-r', '3', 'foo=bar', '--app=False']], - ), ( - lambda: fake_script(self, 'relation-set', 'echo {} >&2 ; exit 2'.format(err_msg)), - lambda: self.backend.relation_set(3, 'foo', 'bar', is_app=False), - ops.model.RelationNotFoundError, - [['relation-set', '-r', '3', 'foo=bar', '--app=False']], - ), ( - lambda: fake_script(self, 'relation-get', 'echo fooerror >&2 ; exit 1'), - lambda: self.backend.relation_get(3, 'remote/0', is_app=False), - ops.model.ModelError, - [['relation-get', '-r', '3', '-', 'remote/0', '--app=False', '--format=json']], - ), ( - lambda: fake_script(self, 'relation-get', 'echo {} >&2 ; exit 2'.format(err_msg)), - lambda: self.backend.relation_get(3, 'remote/0', is_app=False), - ops.model.RelationNotFoundError, - [['relation-get', '-r', '3', '-', 'remote/0', '--app=False', '--format=json']], - )] - - for do_fake, run, exception, calls in test_cases: - do_fake() - with self.assertRaises(exception): - run() - self.assertEqual(fake_script_calls(self, clear=True), calls) - - def test_status_is_app_forced_kwargs(self): - fake_script(self, 'status-get', 'exit 1') - fake_script(self, 'status-set', 'exit 1') - - test_cases = ( - lambda: self.backend.status_get(False), - lambda: self.backend.status_get(True), - lambda: self.backend.status_set('active', '', False), - lambda: self.backend.status_set('active', '', True), - ) - - for case in test_cases: - with self.assertRaises(TypeError): - case() - - def test_storage_tool_errors(self): - test_cases = [( - lambda: fake_script(self, 'storage-list', 'echo fooerror >&2 ; exit 1'), - lambda: self.backend.storage_list('foobar'), - ops.model.ModelError, - [['storage-list', 'foobar', '--format=json']], - ), ( - lambda: fake_script(self, 'storage-get', 'echo fooerror >&2 ; exit 1'), - lambda: self.backend.storage_get('foobar', 'someattr'), - ops.model.ModelError, - [['storage-get', '-s', 'foobar', 'someattr', '--format=json']], - ), ( - lambda: fake_script(self, 'storage-add', 'echo fooerror >&2 ; exit 1'), - lambda: self.backend.storage_add('foobar', count=2), - ops.model.ModelError, - [['storage-add', 'foobar=2']], - ), ( - lambda: fake_script(self, 'storage-add', 'echo fooerror >&2 ; exit 1'), - lambda: self.backend.storage_add('foobar', count=object), - TypeError, - [], - ), ( - lambda: fake_script(self, 'storage-add', 'echo fooerror >&2 ; exit 1'), - lambda: self.backend.storage_add('foobar', count=True), - TypeError, - [], - )] - for do_fake, run, exception, calls in test_cases: - do_fake() - with self.assertRaises(exception): - run() - self.assertEqual(fake_script_calls(self, clear=True), calls) - - def test_network_get(self): - network_get_out = '''{ - "bind-addresses": [ - { - "mac-address": "", - "interface-name": "", - "addresses": [ - { - "hostname": "", - "value": "192.0.2.2", - "cidr": "" - } - ] - } - ], - "egress-subnets": [ - "192.0.2.2/32" - ], - "ingress-addresses": [ - "192.0.2.2" - ] -}''' - fake_script(self, 'network-get', - '''[ "$1" = deadbeef ] && echo '{}' || exit 1'''.format(network_get_out)) - network_info = self.backend.network_get('deadbeef') - self.assertEqual(network_info, json.loads(network_get_out)) - self.assertEqual(fake_script_calls(self, clear=True), - [['network-get', 'deadbeef', '--format=json']]) - - network_info = self.backend.network_get('deadbeef', 1) - self.assertEqual(network_info, json.loads(network_get_out)) - self.assertEqual(fake_script_calls(self, clear=True), - [['network-get', 'deadbeef', '-r', '1', '--format=json']]) - - def test_network_get_errors(self): - err_no_endpoint = 'ERROR no network config found for binding "$2"' - err_no_rel = 'ERROR invalid value "$3" for option -r: relation not found' - - test_cases = [( - lambda: fake_script(self, 'network-get', - 'echo {} >&2 ; exit 1'.format(err_no_endpoint)), - lambda: self.backend.network_get("deadbeef"), - ops.model.ModelError, - [['network-get', 'deadbeef', '--format=json']], - ), ( - lambda: fake_script(self, 'network-get', 'echo {} >&2 ; exit 2'.format(err_no_rel)), - lambda: self.backend.network_get("deadbeef", 3), - ops.model.RelationNotFoundError, - [['network-get', 'deadbeef', '-r', '3', '--format=json']], - )] - for do_fake, run, exception, calls in test_cases: - do_fake() - with self.assertRaises(exception): - run() - self.assertEqual(fake_script_calls(self, clear=True), calls) - - def test_action_get_error(self): - fake_script(self, 'action-get', '') - fake_script(self, 'action-get', 'echo fooerror >&2 ; exit 1') - with self.assertRaises(ops.model.ModelError): - self.backend.action_get() - calls = [['action-get', '--format=json']] - self.assertEqual(fake_script_calls(self, clear=True), calls) - - def test_action_set_error(self): - fake_script(self, 'action-get', '') - fake_script(self, 'action-set', 'echo fooerror >&2 ; exit 1') - with self.assertRaises(ops.model.ModelError): - self.backend.action_set(OrderedDict([('foo', 'bar'), ('dead', 'beef cafe')])) - calls = [["action-set", "foo=bar", "dead=beef cafe"]] - self.assertEqual(fake_script_calls(self, clear=True), calls) - - def test_action_log_error(self): - fake_script(self, 'action-get', '') - fake_script(self, 'action-log', 'echo fooerror >&2 ; exit 1') - with self.assertRaises(ops.model.ModelError): - self.backend.action_log('log-message') - calls = [["action-log", "log-message"]] - self.assertEqual(fake_script_calls(self, clear=True), calls) - - def test_action_get(self): - fake_script(self, 'action-get', """echo '{"foo-name": "bar", "silent": false}'""") - params = self.backend.action_get() - self.assertEqual(params['foo-name'], 'bar') - self.assertEqual(params['silent'], False) - self.assertEqual(fake_script_calls(self), [['action-get', '--format=json']]) - - def test_action_set(self): - fake_script(self, 'action-get', 'exit 1') - fake_script(self, 'action-set', 'exit 0') - self.backend.action_set(OrderedDict([('x', 'dead beef'), ('y', 1)])) - self.assertEqual(fake_script_calls(self), [['action-set', 'x=dead beef', 'y=1']]) - - def test_action_fail(self): - fake_script(self, 'action-get', 'exit 1') - fake_script(self, 'action-fail', 'exit 0') - self.backend.action_fail('error 42') - self.assertEqual(fake_script_calls(self), [['action-fail', 'error 42']]) - - def test_action_log(self): - fake_script(self, 'action-get', 'exit 1') - fake_script(self, 'action-log', 'exit 0') - self.backend.action_log('progress: 42%') - self.assertEqual(fake_script_calls(self), [['action-log', 'progress: 42%']]) - - def test_application_version_set(self): - fake_script(self, 'application-version-set', 'exit 0') - self.backend.application_version_set('1.2b3') - self.assertEqual(fake_script_calls(self), [['application-version-set', '--', '1.2b3']]) - - def test_application_version_set_invalid(self): - fake_script(self, 'application-version-set', 'exit 0') - with self.assertRaises(TypeError): - self.backend.application_version_set(2) - with self.assertRaises(TypeError): - self.backend.application_version_set() - self.assertEqual(fake_script_calls(self), []) - - def test_juju_log(self): - fake_script(self, 'juju-log', 'exit 0') - self.backend.juju_log('WARNING', 'foo') - self.assertEqual(fake_script_calls(self, clear=True), - [['juju-log', '--log-level', 'WARNING', 'foo']]) - - with self.assertRaises(TypeError): - self.backend.juju_log('DEBUG') - self.assertEqual(fake_script_calls(self, clear=True), []) - - fake_script(self, 'juju-log', 'exit 1') - with self.assertRaises(ops.model.ModelError): - self.backend.juju_log('BAR', 'foo') - self.assertEqual(fake_script_calls(self, clear=True), - [['juju-log', '--log-level', 'BAR', 'foo']]) - - def test_valid_metrics(self): - fake_script(self, 'add-metric', 'exit 0') - test_cases = [( - OrderedDict([('foo', 42), ('b-ar', 4.5), ('ba_-z', 4.5), ('a', 1)]), - OrderedDict([('de', 'ad'), ('be', 'ef_ -')]), - [['add-metric', '--labels', 'de=ad,be=ef_ -', - 'foo=42', 'b-ar=4.5', 'ba_-z=4.5', 'a=1']] - ), ( - OrderedDict([('foo1', 0), ('b2r', 4.5)]), - OrderedDict([('d3', 'aд'), ('b33f', '3_ -')]), - [['add-metric', '--labels', 'd3=aд,b33f=3_ -', 'foo1=0', 'b2r=4.5']], - )] - for metrics, labels, expected_calls in test_cases: - self.backend.add_metrics(metrics, labels) - self.assertEqual(fake_script_calls(self, clear=True), expected_calls) - - def test_invalid_metric_names(self): - invalid_inputs = [ - ({'': 4.2}, {}), - ({'1': 4.2}, {}), - ({'1': -4.2}, {}), - ({'123': 4.2}, {}), - ({'1foo': 4.2}, {}), - ({'-foo': 4.2}, {}), - ({'_foo': 4.2}, {}), - ({'foo-': 4.2}, {}), - ({'foo_': 4.2}, {}), - ({'a-': 4.2}, {}), - ({'a_': 4.2}, {}), - ({'BAЯ': 4.2}, {}), - ] - for metrics, labels in invalid_inputs: - with self.assertRaises(ops.model.ModelError): - self.backend.add_metrics(metrics, labels) - - def test_invalid_metric_values(self): - invalid_inputs = [ - ({'a': float('+inf')}, {}), - ({'a': float('-inf')}, {}), - ({'a': float('nan')}, {}), - ({'foo': 'bar'}, {}), - ({'foo': '1O'}, {}), - ] - for metrics, labels in invalid_inputs: - with self.assertRaises(ops.model.ModelError): - self.backend.add_metrics(metrics, labels) - - def test_invalid_metric_labels(self): - invalid_inputs = [ - ({'foo': 4.2}, {'': 'baz'}), - ({'foo': 4.2}, {',bar': 'baz'}), - ({'foo': 4.2}, {'b=a=r': 'baz'}), - ({'foo': 4.2}, {'BAЯ': 'baz'}), - ] - for metrics, labels in invalid_inputs: - with self.assertRaises(ops.model.ModelError): - self.backend.add_metrics(metrics, labels) - - def test_invalid_metric_label_values(self): - invalid_inputs = [ - ({'foo': 4.2}, {'bar': ''}), - ({'foo': 4.2}, {'bar': 'b,az'}), - ({'foo': 4.2}, {'bar': 'b=az'}), - ] - for metrics, labels in invalid_inputs: - with self.assertRaises(ops.model.ModelError): - self.backend.add_metrics(metrics, labels) - - -class TestLazyMapping(unittest.TestCase): - - def test_invalidate(self): - loaded = [] - - class MyLazyMap(ops.model.LazyMapping): - def _load(self): - loaded.append(1) - return {'foo': 'bar'} - - map = MyLazyMap() - self.assertEqual(map['foo'], 'bar') - self.assertEqual(loaded, [1]) - self.assertEqual(map['foo'], 'bar') - self.assertEqual(loaded, [1]) - map._invalidate() - self.assertEqual(map['foo'], 'bar') - self.assertEqual(loaded, [1, 1]) - - -if __name__ == "__main__": - unittest.main() diff --git a/magma/squid_cnf/charms/squid/mod/operator/test/test_testing.py b/magma/squid_cnf/charms/squid/mod/operator/test/test_testing.py deleted file mode 100644 index 1645e5a2a6329c0f6f23e8e2de028051a3783978..0000000000000000000000000000000000000000 --- a/magma/squid_cnf/charms/squid/mod/operator/test/test_testing.py +++ /dev/null @@ -1,756 +0,0 @@ -#!/usr/bin/python3 -# Copyright 2019 Canonical Ltd. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import importlib -import pathlib -import shutil -import sys -import tempfile -import textwrap -import unittest - -from ops.charm import ( - CharmBase, - RelationEvent, -) -from ops.framework import ( - Object, -) -from ops.model import ( - ModelError, - RelationNotFoundError, -) -from ops.testing import Harness - - -class TestHarness(unittest.TestCase): - - def test_add_relation(self): - harness = Harness(CharmBase, meta=''' - name: test-app - requires: - db: - interface: pgsql - ''') - rel_id = harness.add_relation('db', 'postgresql') - self.assertIsInstance(rel_id, int) - backend = harness._backend - self.assertEqual(backend.relation_ids('db'), [rel_id]) - self.assertEqual(backend.relation_list(rel_id), []) - # Make sure the initial data bags for our app and unit are empty. - self.assertEqual(backend.relation_get(rel_id, 'test-app', is_app=True), {}) - self.assertEqual(backend.relation_get(rel_id, 'test-app/0', is_app=False), {}) - - def test_add_relation_and_unit(self): - harness = Harness(CharmBase, meta=''' - name: test-app - requires: - db: - interface: pgsql - ''') - rel_id = harness.add_relation('db', 'postgresql') - self.assertIsInstance(rel_id, int) - harness.add_relation_unit(rel_id, 'postgresql/0') - harness.update_relation_data(rel_id, 'postgresql/0', {'foo': 'bar'}) - backend = harness._backend - self.assertEqual(backend.relation_ids('db'), [rel_id]) - self.assertEqual(backend.relation_list(rel_id), ['postgresql/0']) - self.assertEqual( - backend.relation_get(rel_id, 'postgresql/0', is_app=False), - {'foo': 'bar'}) - - def test_add_relation_with_remote_app_data(self): - # language=YAML - harness = Harness(CharmBase, meta=''' - name: test-app - requires: - db: - interface: pgsql - ''') - remote_app = 'postgresql' - rel_id = harness.add_relation('db', remote_app) - harness.update_relation_data(rel_id, 'postgresql', {'app': 'data'}) - self.assertIsInstance(rel_id, int) - backend = harness._backend - self.assertEqual([rel_id], backend.relation_ids('db')) - self.assertEqual({'app': 'data'}, backend.relation_get(rel_id, remote_app, is_app=True)) - - def test_add_relation_with_our_initial_data(self): - - class InitialDataTester(CharmBase): - """Record the relation-changed events.""" - - def __init__(self, framework, charm_name): - super().__init__(framework, charm_name) - self.observed_events = [] - self.framework.observe(self.on.db_relation_changed, self._on_db_relation_changed) - - def _on_db_relation_changed(self, event): - self.observed_events.append(event) - - # language=YAML - harness = Harness(InitialDataTester, meta=''' - name: test-app - requires: - db: - interface: pgsql - ''') - rel_id = harness.add_relation('db', 'postgresql') - harness.update_relation_data(rel_id, 'test-app', {'k': 'v1'}) - harness.update_relation_data(rel_id, 'test-app/0', {'ingress-address': '192.0.2.1'}) - backend = harness._backend - self.assertEqual({'k': 'v1'}, backend.relation_get(rel_id, 'test-app', is_app=True)) - self.assertEqual({'ingress-address': '192.0.2.1'}, - backend.relation_get(rel_id, 'test-app/0', is_app=False)) - - harness.begin() - self.assertEqual({'k': 'v1'}, backend.relation_get(rel_id, 'test-app', is_app=True)) - self.assertEqual({'ingress-address': '192.0.2.1'}, - backend.relation_get(rel_id, 'test-app/0', is_app=False)) - # Make sure no relation-changed events are emitted for our own data bags. - self.assertEqual([], harness.charm.observed_events) - - # A remote unit can still update our app relation data bag since our unit is not a leader. - harness.update_relation_data(rel_id, 'test-app', {'k': 'v2'}) - # And we get an event - self.assertEqual([], harness.charm.observed_events) - # We can also update our own relation data, even if it is a bit 'cheaty' - harness.update_relation_data(rel_id, 'test-app/0', {'ingress-address': '192.0.2.2'}) - # But no event happens - - # Updating our data app relation data bag and our unit data bag does not generate events. - harness.set_leader(True) - harness.update_relation_data(rel_id, 'test-app', {'k': 'v3'}) - harness.update_relation_data(rel_id, 'test-app/0', {'ingress-address': '192.0.2.2'}) - self.assertEqual([], harness.charm.observed_events) - - def test_add_peer_relation_with_initial_data_leader(self): - - class InitialDataTester(CharmBase): - """Record the relation-changed events.""" - - def __init__(self, framework, charm_name): - super().__init__(framework, charm_name) - self.observed_events = [] - self.framework.observe(self.on.cluster_relation_changed, - self._on_cluster_relation_changed) - - def _on_cluster_relation_changed(self, event): - self.observed_events.append(event) - - # language=YAML - harness = Harness(InitialDataTester, meta=''' - name: test-app - peers: - cluster: - interface: cluster - ''') - # TODO: dmitriis 2020-04-07 test a minion unit and initial peer relation app data - # events when the harness begins to emit events for initial data. - harness.set_leader(is_leader=True) - rel_id = harness.add_relation('cluster', 'test-app') - harness.update_relation_data(rel_id, 'test-app', {'k': 'v'}) - harness.update_relation_data(rel_id, 'test-app/0', {'ingress-address': '192.0.2.1'}) - backend = harness._backend - self.assertEqual({'k': 'v'}, backend.relation_get(rel_id, 'test-app', is_app=True)) - self.assertEqual({'ingress-address': '192.0.2.1'}, - backend.relation_get(rel_id, 'test-app/0', is_app=False)) - - harness.begin() - self.assertEqual({'k': 'v'}, backend.relation_get(rel_id, 'test-app', is_app=True)) - self.assertEqual({'ingress-address': '192.0.2.1'}, - backend.relation_get(rel_id, 'test-app/0', is_app=False)) - # Make sure no relation-changed events are emitted for our own data bags. - self.assertEqual([], harness.charm.observed_events) - - # Updating our app relation data bag and our unit data bag does not trigger events - harness.update_relation_data(rel_id, 'test-app', {'k': 'v2'}) - harness.update_relation_data(rel_id, 'test-app/0', {'ingress-address': '192.0.2.2'}) - self.assertEqual([], harness.charm.observed_events) - - # If our unit becomes a minion, updating app relation data indirectly becomes possible - # and our charm gets notifications. - harness.set_leader(False) - harness.update_relation_data(rel_id, 'test-app', {'k': 'v3'}) - self.assertEqual({'k': 'v3'}, backend.relation_get(rel_id, 'test-app', is_app=True)) - self.assertTrue(len(harness.charm.observed_events), 1) - self.assertIsInstance(harness.charm.observed_events[0], RelationEvent) - - def test_relation_events(self): - harness = Harness(RelationEventCharm, meta=''' - name: test-app - requires: - db: - interface: pgsql - ''') - harness.begin() - harness.charm.observe_relation_events('db') - self.assertEqual(harness.charm.get_changes(), []) - rel_id = harness.add_relation('db', 'postgresql') - self.assertEqual( - harness.charm.get_changes(), - [{'name': 'relation-created', - 'data': { - 'app': 'postgresql', - 'unit': None, - 'relation_id': rel_id, - }}]) - harness.add_relation_unit(rel_id, 'postgresql/0') - self.assertEqual( - harness.charm.get_changes(), - [{'name': 'relation-joined', - 'data': { - 'app': 'postgresql', - 'unit': 'postgresql/0', - 'relation_id': rel_id, - }}]) - harness.update_relation_data(rel_id, 'postgresql', {'foo': 'bar'}) - self.assertEqual( - harness.charm.get_changes(), - [{'name': 'relation-changed', - 'data': { - 'app': 'postgresql', - 'unit': None, - 'relation_id': rel_id, - }}]) - harness.update_relation_data(rel_id, 'postgresql/0', {'baz': 'bing'}) - self.assertEqual( - harness.charm.get_changes(), - [{'name': 'relation-changed', - 'data': { - 'app': 'postgresql', - 'unit': 'postgresql/0', - 'relation_id': rel_id, - }}]) - - def test_get_relation_data(self): - harness = Harness(CharmBase, meta=''' - name: test-app - requires: - db: - interface: pgsql - ''') - rel_id = harness.add_relation('db', 'postgresql') - harness.update_relation_data(rel_id, 'postgresql', {'remote': 'data'}) - self.assertEqual(harness.get_relation_data(rel_id, 'test-app'), {}) - self.assertEqual(harness.get_relation_data(rel_id, 'test-app/0'), {}) - self.assertEqual(harness.get_relation_data(rel_id, 'test-app/1'), None) - self.assertEqual(harness.get_relation_data(rel_id, 'postgresql'), {'remote': 'data'}) - with self.assertRaises(KeyError): - # unknown relation id - harness.get_relation_data(99, 'postgresql') - - def test_create_harness_twice(self): - metadata = ''' - name: my-charm - requires: - db: - interface: pgsql - ''' - harness1 = Harness(CharmBase, meta=metadata) - harness2 = Harness(CharmBase, meta=metadata) - harness1.begin() - harness2.begin() - helper1 = DBRelationChangedHelper(harness1.charm, "helper1") - helper2 = DBRelationChangedHelper(harness2.charm, "helper2") - rel_id = harness2.add_relation('db', 'postgresql') - harness2.update_relation_data(rel_id, 'postgresql', {'key': 'value'}) - # Helper2 should see the event triggered by harness2, but helper1 should see no events. - self.assertEqual(helper1.changes, []) - self.assertEqual(helper2.changes, [(rel_id, 'postgresql')]) - - def test_begin_twice(self): - # language=YAML - harness = Harness(CharmBase, meta=''' - name: test-app - requires: - db: - interface: pgsql - ''') - harness.begin() - with self.assertRaises(RuntimeError): - harness.begin() - - def test_update_relation_exposes_new_data(self): - harness = Harness(CharmBase, meta=''' - name: my-charm - requires: - db: - interface: pgsql - ''') - harness.begin() - viewer = RelationChangedViewer(harness.charm, 'db') - rel_id = harness.add_relation('db', 'postgresql') - harness.add_relation_unit(rel_id, 'postgresql/0') - harness.update_relation_data(rel_id, 'postgresql/0', {'initial': 'data'}) - self.assertEqual(viewer.changes, [{'initial': 'data'}]) - harness.update_relation_data(rel_id, 'postgresql/0', {'new': 'value'}) - self.assertEqual(viewer.changes, [{'initial': 'data'}, - {'initial': 'data', 'new': 'value'}]) - - def test_update_relation_no_local_unit_change_event(self): - # language=YAML - harness = Harness(CharmBase, meta=''' - name: my-charm - requires: - db: - interface: pgsql - ''') - harness.begin() - helper = DBRelationChangedHelper(harness.charm, "helper") - rel_id = harness.add_relation('db', 'postgresql') - rel = harness.charm.model.get_relation('db') - rel.data[harness.charm.model.unit]['key'] = 'value' - # there should be no event for updating our own data - harness.update_relation_data(rel_id, 'my-charm/0', {'new': 'other'}) - # but the data will be updated. - self.assertEqual({'key': 'value', 'new': 'other'}, rel.data[harness.charm.model.unit]) - - rel.data[harness.charm.model.unit]['new'] = 'value' - # Our unit data bag got updated. - self.assertEqual(rel.data[harness.charm.model.unit]['new'], 'value') - # But there were no changed events registered by our unit. - self.assertEqual([], helper.changes) - - def test_update_peer_relation_no_local_unit_change_event(self): - # language=YAML - harness = Harness(CharmBase, meta=''' - name: postgresql - peers: - db: - interface: pgsql - ''') - harness.begin() - helper = DBRelationChangedHelper(harness.charm, "helper") - rel_id = harness.add_relation('db', 'postgresql') - - rel = harness.charm.model.get_relation('db') - rel.data[harness.charm.model.unit]['key'] = 'value' - rel = harness.charm.model.get_relation('db') - harness.update_relation_data(rel_id, 'postgresql/0', {'key': 'v1'}) - self.assertEqual({'key': 'v1'}, rel.data[harness.charm.model.unit]) - # Make sure there was no event - self.assertEqual([], helper.changes) - - rel.data[harness.charm.model.unit]['key'] = 'v2' - # Our unit data bag got updated. - self.assertEqual({'key': 'v2'}, dict(rel.data[harness.charm.model.unit])) - # But there were no changed events registered by our unit. - self.assertEqual([], helper.changes) - - # Same for when our unit is a leader. - harness.set_leader(is_leader=True) - harness.update_relation_data(rel_id, 'postgresql/0', {'key': 'v3'}) - self.assertEqual({'key': 'v3'}, dict(rel.data[harness.charm.model.unit])) - self.assertEqual([], helper.changes) - - rel.data[harness.charm.model.unit]['key'] = 'v4' - self.assertEqual(rel.data[harness.charm.model.unit]['key'], 'v4') - self.assertEqual([], helper.changes) - - def test_update_peer_relation_app_data(self): - # language=YAML - harness = Harness(CharmBase, meta=''' - name: postgresql - peers: - db: - interface: pgsql - ''') - harness.begin() - harness.set_leader(is_leader=True) - helper = DBRelationChangedHelper(harness.charm, "helper") - rel_id = harness.add_relation('db', 'postgresql') - rel = harness.charm.model.get_relation('db') - rel.data[harness.charm.app]['key'] = 'value' - harness.update_relation_data(rel_id, 'postgresql', {'key': 'v1'}) - self.assertEqual({'key': 'v1'}, rel.data[harness.charm.app]) - self.assertEqual([], helper.changes) - - rel.data[harness.charm.app]['key'] = 'v2' - # Our unit data bag got updated. - self.assertEqual(rel.data[harness.charm.model.app]['key'], 'v2') - # But there were no changed events registered by our unit. - self.assertEqual([], helper.changes) - - # If our unit is not a leader unit we get an update about peer app relation data changes. - harness.set_leader(is_leader=False) - harness.update_relation_data(rel_id, 'postgresql', {'k2': 'v2'}) - self.assertEqual(rel.data[harness.charm.model.app]['k2'], 'v2') - self.assertEqual(helper.changes, [(0, 'postgresql')]) - - def test_update_relation_no_local_app_change_event(self): - # language=YAML - harness = Harness(CharmBase, meta=''' - name: my-charm - requires: - db: - interface: pgsql - ''') - harness.begin() - harness.set_leader(False) - helper = DBRelationChangedHelper(harness.charm, "helper") - rel_id = harness.add_relation('db', 'postgresql') - # TODO: remove this as soon as https://github.com/canonical/operator/issues/175 is fixed. - harness.add_relation_unit(rel_id, 'postgresql/0') - self.assertEqual(helper.changes, []) - - harness.update_relation_data(rel_id, 'my-charm', {'new': 'value'}) - rel = harness.charm.model.get_relation('db') - self.assertEqual(rel.data[harness.charm.app]['new'], 'value') - - # Our app data bag got updated. - self.assertEqual(rel.data[harness.charm.model.app]['new'], 'value') - # But there were no changed events registered by our unit. - self.assertEqual(helper.changes, []) - - def test_update_relation_remove_data(self): - harness = Harness(CharmBase, meta=''' - name: my-charm - requires: - db: - interface: pgsql - ''') - harness.begin() - viewer = RelationChangedViewer(harness.charm, 'db') - rel_id = harness.add_relation('db', 'postgresql') - harness.add_relation_unit(rel_id, 'postgresql/0') - harness.update_relation_data(rel_id, 'postgresql/0', {'initial': 'data'}) - harness.update_relation_data(rel_id, 'postgresql/0', {'initial': ''}) - self.assertEqual(viewer.changes, [{'initial': 'data'}, {}]) - - def test_update_config(self): - harness = Harness(RecordingCharm) - harness.begin() - harness.update_config(key_values={'a': 'foo', 'b': 2}) - self.assertEqual( - harness.charm.changes, - [{'name': 'config', 'data': {'a': 'foo', 'b': 2}}]) - harness.update_config(key_values={'b': 3}) - self.assertEqual( - harness.charm.changes, - [{'name': 'config', 'data': {'a': 'foo', 'b': 2}}, - {'name': 'config', 'data': {'a': 'foo', 'b': 3}}]) - # you can set config values to the empty string, you can use unset to actually remove items - harness.update_config(key_values={'a': ''}, unset=set('b')) - self.assertEqual( - harness.charm.changes, - [{'name': 'config', 'data': {'a': 'foo', 'b': 2}}, - {'name': 'config', 'data': {'a': 'foo', 'b': 3}}, - {'name': 'config', 'data': {'a': ''}}, - ]) - - def test_set_leader(self): - harness = Harness(RecordingCharm) - # No event happens here - harness.set_leader(False) - harness.begin() - self.assertFalse(harness.charm.model.unit.is_leader()) - harness.set_leader(True) - self.assertEqual(harness.charm.get_changes(reset=True), [{'name': 'leader-elected'}]) - self.assertTrue(harness.charm.model.unit.is_leader()) - harness.set_leader(False) - self.assertFalse(harness.charm.model.unit.is_leader()) - # No hook event when you lose leadership. - # TODO: verify if Juju always triggers `leader-settings-changed` if you - # lose leadership. - self.assertEqual(harness.charm.get_changes(reset=True), []) - harness.disable_hooks() - harness.set_leader(True) - # No hook event if you have disabled them - self.assertEqual(harness.charm.get_changes(reset=True), []) - - def test_relation_set_app_not_leader(self): - harness = Harness(RecordingCharm, meta=''' - name: test-charm - requires: - db: - interface: pgsql - ''') - harness.begin() - harness.set_leader(False) - rel_id = harness.add_relation('db', 'postgresql') - harness.add_relation_unit(rel_id, 'postgresql/0') - rel = harness.charm.model.get_relation('db') - with self.assertRaises(ModelError): - rel.data[harness.charm.app]['foo'] = 'bar' - # The data has not actually been changed - self.assertEqual(harness.get_relation_data(rel_id, 'test-charm'), {}) - harness.set_leader(True) - rel.data[harness.charm.app]['foo'] = 'bar' - self.assertEqual(harness.get_relation_data(rel_id, 'test-charm'), {'foo': 'bar'}) - - def test_hooks_enabled_and_disabled(self): - harness = Harness(RecordingCharm, meta=''' - name: test-charm - ''') - # Before begin() there are no events. - harness.update_config({'value': 'first'}) - # By default, after begin the charm is set up to receive events. - harness.begin() - harness.update_config({'value': 'second'}) - self.assertEqual( - harness.charm.get_changes(reset=True), - [{'name': 'config', 'data': {'value': 'second'}}]) - # Once disabled, we won't see config-changed when we make an update - harness.disable_hooks() - harness.update_config({'third': '3'}) - self.assertEqual(harness.charm.get_changes(reset=True), []) - harness.enable_hooks() - harness.update_config({'value': 'fourth'}) - self.assertEqual( - harness.charm.get_changes(reset=True), - [{'name': 'config', 'data': {'value': 'fourth', 'third': '3'}}]) - - def test_metadata_from_directory(self): - tmp = pathlib.Path(tempfile.mkdtemp()) - self.addCleanup(shutil.rmtree, str(tmp)) - metadata_filename = tmp / 'metadata.yaml' - with metadata_filename.open('wt') as metadata: - metadata.write(textwrap.dedent(''' - name: my-charm - requires: - db: - interface: pgsql - ''')) - harness = self._get_dummy_charm_harness(tmp) - harness.begin() - self.assertEqual(list(harness.model.relations), ['db']) - # The charm_dir also gets set - self.assertEqual(harness.framework.charm_dir, tmp) - - def test_actions_from_directory(self): - tmp = pathlib.Path(tempfile.mkdtemp()) - self.addCleanup(shutil.rmtree, str(tmp)) - actions_filename = tmp / 'actions.yaml' - with actions_filename.open('wt') as actions: - actions.write(textwrap.dedent(''' - test: - description: a dummy action - ''')) - harness = self._get_dummy_charm_harness(tmp) - harness.begin() - self.assertEqual(list(harness.framework.meta.actions), ['test']) - # The charm_dir also gets set - self.assertEqual(harness.framework.charm_dir, tmp) - - def _get_dummy_charm_harness(self, tmp): - self._write_dummy_charm(tmp) - charm_mod = importlib.import_module('charm') - harness = Harness(charm_mod.MyTestingCharm) - return harness - - def _write_dummy_charm(self, tmp): - srcdir = tmp / 'src' - srcdir.mkdir(0o755) - charm_filename = srcdir / 'charm.py' - with charm_filename.open('wt') as charmpy: - # language=Python - charmpy.write(textwrap.dedent(''' - from ops.charm import CharmBase - class MyTestingCharm(CharmBase): - pass - ''')) - orig = sys.path[:] - sys.path.append(str(srcdir)) - - def cleanup(): - sys.path = orig - sys.modules.pop('charm') - - self.addCleanup(cleanup) - - def test_actions_passed_in(self): - harness = Harness( - CharmBase, - meta=''' - name: test-app - ''', - actions=''' - test-action: - description: a dummy test action - ''') - self.assertEqual(list(harness.framework.meta.actions), ['test-action']) - - def test_relation_set_deletes(self): - harness = Harness(CharmBase, meta=''' - name: test-charm - requires: - db: - interface: pgsql - ''') - harness.begin() - harness.set_leader(False) - rel_id = harness.add_relation('db', 'postgresql') - harness.update_relation_data(rel_id, 'test-charm/0', {'foo': 'bar'}) - harness.add_relation_unit(rel_id, 'postgresql/0') - rel = harness.charm.model.get_relation('db', rel_id) - del rel.data[harness.charm.model.unit]['foo'] - self.assertEqual({}, harness.get_relation_data(rel_id, 'test-charm/0')) - - def test_set_workload_version(self): - harness = Harness(CharmBase, meta=''' - name: app - ''') - harness.begin() - self.assertIsNone(harness.get_workload_version()) - harness.charm.model.unit.set_workload_version('1.2.3') - self.assertEqual(harness.get_workload_version(), '1.2.3') - - -class DBRelationChangedHelper(Object): - def __init__(self, parent, key): - super().__init__(parent, key) - self.changes = [] - parent.framework.observe(parent.on.db_relation_changed, self.on_relation_changed) - - def on_relation_changed(self, event): - if event.unit is not None: - self.changes.append((event.relation.id, event.unit.name)) - else: - self.changes.append((event.relation.id, event.app.name)) - - -class RelationChangedViewer(Object): - """Track relation_changed events and saves the data seen in the relation bucket.""" - - def __init__(self, charm, relation_name): - super().__init__(charm, relation_name) - self.changes = [] - charm.framework.observe(charm.on[relation_name].relation_changed, self.on_relation_changed) - - def on_relation_changed(self, event): - if event.unit is not None: - data = event.relation.data[event.unit] - else: - data = event.relation.data[event.app] - self.changes.append(dict(data)) - - -class RecordingCharm(CharmBase): - """Record the events that we see, and any associated data.""" - - def __init__(self, framework, charm_name): - super().__init__(framework, charm_name) - self.changes = [] - self.framework.observe(self.on.config_changed, self.on_config_changed) - self.framework.observe(self.on.leader_elected, self.on_leader_elected) - - def get_changes(self, reset=True): - changes = self.changes - if reset: - self.changes = [] - return changes - - def on_config_changed(self, _): - self.changes.append(dict(name='config', data=dict(self.framework.model.config))) - - def on_leader_elected(self, _): - self.changes.append(dict(name='leader-elected')) - - -class RelationEventCharm(RecordingCharm): - """Record events related to relation lifecycles.""" - - def __init__(self, framework, charm_name): - super().__init__(framework, charm_name) - - def observe_relation_events(self, relation_name): - self.framework.observe(self.on[relation_name].relation_created, self._on_relation_created) - self.framework.observe(self.on[relation_name].relation_joined, self._on_relation_joined) - self.framework.observe(self.on[relation_name].relation_changed, self._on_relation_changed) - self.framework.observe(self.on[relation_name].relation_departed, - self._on_relation_departed) - self.framework.observe(self.on[relation_name].relation_broken, self._on_relation_broken) - - def _on_relation_created(self, event): - self._observe_relation_event('relation-created', event) - - def _on_relation_joined(self, event): - self._observe_relation_event('relation-joined', event) - - def _on_relation_changed(self, event): - self._observe_relation_event('relation-changed', event) - - def _on_relation_departed(self, event): - self._observe_relation_event('relation-departed', event) - - def _on_relation_broken(self, event): - self._observe_relation_event('relation-broken', event) - - def _observe_relation_event(self, event_name, event): - unit_name = None - if event.unit is not None: - unit_name = event.unit.name - app_name = None - if event.app is not None: - app_name = event.app.name - self.changes.append( - dict(name=event_name, - data=dict(app=app_name, unit=unit_name, relation_id=event.relation.id))) - - -class TestTestingModelBackend(unittest.TestCase): - - def test_status_set_get_unit(self): - harness = Harness(CharmBase, meta=''' - name: app - ''') - backend = harness._backend - backend.status_set('blocked', 'message', is_app=False) - self.assertEqual(backend.status_get(is_app=False), ('blocked', 'message')) - self.assertEqual(backend.status_get(is_app=True), None) - - def test_status_set_get_app(self): - harness = Harness(CharmBase, meta=''' - name: app - ''') - backend = harness._backend - backend.status_set('blocked', 'message', is_app=True) - self.assertEqual(backend.status_get(is_app=True), ('blocked', 'message')) - self.assertEqual(backend.status_get(is_app=False), None) - - def test_relation_ids_unknown_relation(self): - harness = Harness(CharmBase, meta=''' - name: test-charm - provides: - db: - interface: mydb - ''') - backend = harness._backend - # With no relations added, we just get an empty list for the interface - self.assertEqual(backend.relation_ids('db'), []) - # But an unknown interface raises a ModelError - with self.assertRaises(ModelError): - backend.relation_ids('unknown') - - def test_relation_get_unknown_relation_id(self): - harness = Harness(CharmBase, meta=''' - name: test-charm - ''') - backend = harness._backend - with self.assertRaises(RelationNotFoundError): - backend.relation_get(1234, 'unit/0', False) - - def test_relation_list_unknown_relation_id(self): - harness = Harness(CharmBase, meta=''' - name: test-charm - ''') - backend = harness._backend - with self.assertRaises(RelationNotFoundError): - backend.relation_list(1234) - - -if __name__ == "__main__": - unittest.main() diff --git a/magma/squid_cnf/charms/squid/requirements.txt b/magma/squid_cnf/charms/squid/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..4da6f5fe4169cdf33fb85954f2ea3db2290f3fc8 --- /dev/null +++ b/magma/squid_cnf/charms/squid/requirements.txt @@ -0,0 +1,3 @@ +ops +jinja2 +git+https://github.com/juju-solutions/resource-oci-image/@c5778285d332edf3d9a538f9d0c06154b7ec1b0b#egg=oci-image diff --git a/magma/squid_cnf/charms/squid/squid.charm b/magma/squid_cnf/charms/squid/squid.charm new file mode 100644 index 0000000000000000000000000000000000000000..00ca09ed1e2947b3e934e45bf8fd81abefdbbc5a Binary files /dev/null and b/magma/squid_cnf/charms/squid/squid.charm differ diff --git a/magma/squid_cnf/charms/squid/src/charm.py b/magma/squid_cnf/charms/squid/src/charm.py index 663410e5323d210ecadb3c5c683c8bbf6d1121c2..e861c025273141d723b5d82f14d59b5b4f997a94 100755 --- a/magma/squid_cnf/charms/squid/src/charm.py +++ b/magma/squid_cnf/charms/squid/src/charm.py @@ -1,120 +1,114 @@ #! /usr/bin/env python3 -# -*- coding: utf-8 -*- -# vim:fenc=utf-8 -# Copyright © 2020 Dominik Fleischmann dominik.fleischmann@canonical.com -"""Operator Charm main library.""" -# Load modules from lib directory import logging +import subprocess -import setuppath # noqa:F401 +from ops.main import main from ops.charm import CharmBase from ops.framework import StoredState from ops.main import main -from ops.model import ActiveStatus, MaintenanceStatus -import subprocess +from ops.model import ActiveStatus, MaintenanceStatus, BlockedStatus + +from oci_image import OCIImageResource, OCIImageResourceError + +from jinja2 import Template + +SQUID_CONF = "/etc/squid/squid.conf" + +logger = logging.getLogger(__name__) class SquidK8SCharm(CharmBase): """Class reprisenting this Operator charm.""" - state = StoredState() + _stored = StoredState() def __init__(self, *args): """Initialize charm and configure states and events to observe.""" super().__init__(*args) - # -- standard hook observation - self.framework.observe(self.on.install, self.on_install) - self.framework.observe(self.on.start, self.on_start) - self.framework.observe(self.on.config_changed, self.on_config_changed) - self.framework.observe(self.on.deleteurl_action, self) - # -- initialize states -- - self.state.set_default(installed=False) - self.state.set_default(configured=False) - self.state.set_default(started=False) - - def make_pod_spec(self): - config = self.framework.model.config - ports = [{"name": "squid", "containerPort": config["port"], "protocol": "TCP"}] - - spec = { - "containers": [{ - "name": self.framework.model.app.name, - "image": config["image"], - "ports": ports, - }], - } - - return spec - - def _apply_spec(self, spec): - # Only apply the spec if this unit is a leader - if self.framework.model.unit.is_leader(): - self.framework.model.pod.set_spec(spec) - self.state.spec = spec - - def on_install(self, event): - """Handle install state.""" - self.unit.status = MaintenanceStatus("Installing charm software") - # Perform install tasks - self.unit.status = MaintenanceStatus("Install complete") - logging.info("Install of software complete") - self.state.installed = True - - def on_config_changed(self, event): - """Handle config changed.""" - - if not self.state.installed: - logging.warning("Config changed called before install complete, deferring event: {}.".format(event.handle)) - - return - - if self.state.started: - # Stop if necessary for reconfig - logging.info("Stopping for configuration, event handle: {}".format(event.handle)) - # Configure the software - logging.info("Configuring") - self.state.configured = True - - def on_start(self, event): - """Handle start state.""" - - if not self.state.configured: - logging.warning("Start called before configuration complete, deferring event: {}".format(event.handle)) + self._stored.set_default(pod_spec=None, allowedurls=set()) + + self.framework.observe(self.on.start, self.configure_pod) + self.framework.observe(self.on.config_changed, self.configure_pod) + self.framework.observe(self.on.addurl_action, self.on_addurl_action) + self.framework.observe(self.on.deleteurl_action, self.on_deleteurl_action) + + self.image = OCIImageResource(self, "image") + + def _update_allowed_urls(self, add: str = None, delete: str = None): + if add: + self._stored.allowedurls.add(add) + if delete and delete in self._stored.allowedurls: + self._stored.allowedurls.remove(delete) + + def _update_squid_config(self, add: str = None, delete: str = None): + self._update_allowed_urls(add=add, delete=delete) + squid_config_text = self._get_squid_config_file_text() + if squid_config_text: + with open(SQUID_CONF, "w") as f: + f.write(squid_config_text) + subprocess.Popen( + "sleep 1 && kill -HUP `cat /var/run/squid.pid`", shell=True + ) + + def on_addurl_action(self, event): + url = event.params["url"] + self._update_squid_config(add=url) + def on_deleteurl_action(self, event): + """Handle the deleteurl action.""" + url = event.params["url"] + self._update_squid_config(delete=url) + + def _get_squid_config_file_text(self): + squid_config_text = None + allowed_urls_text = "" + for url in self._stored.allowedurls: + allowed_urls_text += f"acl allowedurls dstdomain .{url}\n" + allowed_urls_text += "http_access allow allowedurls\n" + with open("template/squid.conf") as template: + squid_config_text = Template(template.read()).render( + allowed_urls=allowed_urls_text + ) + return squid_config_text + + def configure_pod(self, event): + if not self.unit.is_leader(): + self.unit.status = ActiveStatus("ready") return self.unit.status = MaintenanceStatus("Applying pod spec") - # Start software - new_pod_spec = self.make_pod_spec() - self._apply_spec(new_pod_spec) - self.unit.status = ActiveStatus("Unit is ready") - self.state.started = True - logging.info("Started") + # Fetch image information + try: + self.unit.status = MaintenanceStatus("Fetching image information") + image_info = self.image.fetch() + except OCIImageResourceError: + self.unit.status = BlockedStatus("Error fetching image information") + return - def on_deleteurl_action(self, event): - """Handle the deleteurl action.""" - url = event.params["url"] + pod_spec = self._make_pod_spec(image_info) - line_to_delete = "acl allowedurls dstdomain .{}".format(url) - line_deleted = False + if self._stored.pod_spec != pod_spec: + self.model.pod.set_spec(pod_spec) + self._stored.pod_spec = pod_spec + self.unit.status = ActiveStatus("ready") - with open("/etc/squid/squid.conf", "r") as f: - lines = f.readlines() - with open("/etc/squid/squid.conf", "w") as f: - for line in lines: - if line_to_delete not in line: - f.write(line) - else: - line_deleted = True + def _make_pod_spec(self, image_info): + config = self.config + ports = [{"name": "squid", "containerPort": config["port"], "protocol": "TCP"}] - if line_deleted: - event.set_results({"output": "URL deleted succesfully"}) - subprocess.Popen("sleep 1 && kill -HUP `cat /var/run/squid.pid`", shell=True) - else: - event.fail("No URL was deleted") + spec = { + "version": 3, + "containers": [ + { + "name": self.framework.model.app.name, + "imageDetails": image_info, + "ports": ports, + } + ], + } + return spec if __name__ == "__main__": - from ops.main import main main(SquidK8SCharm) diff --git a/magma/squid_cnf/charms/squid/template/squid.conf b/magma/squid_cnf/charms/squid/template/squid.conf new file mode 100644 index 0000000000000000000000000000000000000000..9dc7e8b664d007c9d6dc452dffdc7fe47358d1b2 --- /dev/null +++ b/magma/squid_cnf/charms/squid/template/squid.conf @@ -0,0 +1,26 @@ +acl SSL_ports port 443 +acl Safe_ports port 80 # http +acl Safe_ports port 21 # ftp +acl Safe_ports port 443 # https +acl Safe_ports port 70 # gopher +acl Safe_ports port 210 # wais +acl Safe_ports port 1025-65535 # unregistered ports +acl Safe_ports port 280 # http-mgmt +acl Safe_ports port 488 # gss-http +acl Safe_ports port 591 # filemaker +acl Safe_ports port 777 # multiling http +acl CONNECT method CONNECT +http_access deny !Safe_ports +http_access deny CONNECT !SSL_ports +http_access allow localhost manager +http_access deny manager +http_access allow localhost +{{ allowed_urls }} +http_access deny all +http_port 3128 +coredump_dir /var/spool/squid +refresh_pattern ^ftp: 1440 20% 10080 +refresh_pattern ^gopher: 1440 0% 1440 +refresh_pattern -i (/cgi-bin/|\?) 0 0% 0 +refresh_pattern (Release|Packages(.gz)*)$ 0 20% 2880 +refresh_pattern . 0 20% 4320 diff --git a/magma/squid_cnf/charms/squid/venv/Jinja2-2.11.2.dist-info/INSTALLER b/magma/squid_cnf/charms/squid/venv/Jinja2-2.11.2.dist-info/INSTALLER new file mode 100644 index 0000000000000000000000000000000000000000..a1b589e38a32041e49332e5e81c2d363dc418d68 --- /dev/null +++ b/magma/squid_cnf/charms/squid/venv/Jinja2-2.11.2.dist-info/INSTALLER @@ -0,0 +1 @@ +pip diff --git a/magma/squid_cnf/charms/squid/venv/Jinja2-2.11.2.dist-info/LICENSE.rst b/magma/squid_cnf/charms/squid/venv/Jinja2-2.11.2.dist-info/LICENSE.rst new file mode 100644 index 0000000000000000000000000000000000000000..c37cae49ec77ad6ebb25568c1605f1fee5313cfb --- /dev/null +++ b/magma/squid_cnf/charms/squid/venv/Jinja2-2.11.2.dist-info/LICENSE.rst @@ -0,0 +1,28 @@ +Copyright 2007 Pallets + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A +PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED +TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/magma/squid_cnf/charms/squid/venv/Jinja2-2.11.2.dist-info/METADATA b/magma/squid_cnf/charms/squid/venv/Jinja2-2.11.2.dist-info/METADATA new file mode 100644 index 0000000000000000000000000000000000000000..55c0f82692c5a56768a741095b7d4fb3872b4df5 --- /dev/null +++ b/magma/squid_cnf/charms/squid/venv/Jinja2-2.11.2.dist-info/METADATA @@ -0,0 +1,106 @@ +Metadata-Version: 2.1 +Name: Jinja2 +Version: 2.11.2 +Summary: A very fast and expressive template engine. +Home-page: https://palletsprojects.com/p/jinja/ +Author: Armin Ronacher +Author-email: armin.ronacher@active-4.com +Maintainer: Pallets +Maintainer-email: contact@palletsprojects.com +License: BSD-3-Clause +Project-URL: Documentation, https://jinja.palletsprojects.com/ +Project-URL: Code, https://github.com/pallets/jinja +Project-URL: Issue tracker, https://github.com/pallets/jinja/issues +Platform: UNKNOWN +Classifier: Development Status :: 5 - Production/Stable +Classifier: Environment :: Web Environment +Classifier: Intended Audience :: Developers +Classifier: License :: OSI Approved :: BSD License +Classifier: Operating System :: OS Independent +Classifier: Programming Language :: Python +Classifier: Programming Language :: Python :: 2 +Classifier: Programming Language :: Python :: 2.7 +Classifier: Programming Language :: Python :: 3 +Classifier: Programming Language :: Python :: 3.5 +Classifier: Programming Language :: Python :: 3.6 +Classifier: Programming Language :: Python :: 3.7 +Classifier: Programming Language :: Python :: 3.8 +Classifier: Programming Language :: Python :: Implementation :: CPython +Classifier: Programming Language :: Python :: Implementation :: PyPy +Classifier: Topic :: Internet :: WWW/HTTP :: Dynamic Content +Classifier: Topic :: Software Development :: Libraries :: Python Modules +Classifier: Topic :: Text Processing :: Markup :: HTML +Requires-Python: >=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.* +Description-Content-Type: text/x-rst +Requires-Dist: MarkupSafe (>=0.23) +Provides-Extra: i18n +Requires-Dist: Babel (>=0.8) ; extra == 'i18n' + +Jinja +===== + +Jinja is a fast, expressive, extensible templating engine. Special +placeholders in the template allow writing code similar to Python +syntax. Then the template is passed data to render the final document. + +It includes: + +- Template inheritance and inclusion. +- Define and import macros within templates. +- HTML templates can use autoescaping to prevent XSS from untrusted + user input. +- A sandboxed environment can safely render untrusted templates. +- AsyncIO support for generating templates and calling async + functions. +- I18N support with Babel. +- Templates are compiled to optimized Python code just-in-time and + cached, or can be compiled ahead-of-time. +- Exceptions point to the correct line in templates to make debugging + easier. +- Extensible filters, tests, functions, and even syntax. + +Jinja's philosophy is that while application logic belongs in Python if +possible, it shouldn't make the template designer's job difficult by +restricting functionality too much. + + +Installing +---------- + +Install and update using `pip`_: + +.. code-block:: text + + $ pip install -U Jinja2 + +.. _pip: https://pip.pypa.io/en/stable/quickstart/ + + +In A Nutshell +------------- + +.. code-block:: jinja + + {% extends "base.html" %} + {% block title %}Members{% endblock %} + {% block content %} + + {% endblock %} + + +Links +----- + +- Website: https://palletsprojects.com/p/jinja/ +- Documentation: https://jinja.palletsprojects.com/ +- Releases: https://pypi.org/project/Jinja2/ +- Code: https://github.com/pallets/jinja +- Issue tracker: https://github.com/pallets/jinja/issues +- Test status: https://dev.azure.com/pallets/jinja/_build +- Official chat: https://discord.gg/t6rrQZH + + diff --git a/magma/squid_cnf/charms/squid/venv/Jinja2-2.11.2.dist-info/RECORD b/magma/squid_cnf/charms/squid/venv/Jinja2-2.11.2.dist-info/RECORD new file mode 100644 index 0000000000000000000000000000000000000000..413fef4bfdf0330f017450293400e102cab65113 --- /dev/null +++ b/magma/squid_cnf/charms/squid/venv/Jinja2-2.11.2.dist-info/RECORD @@ -0,0 +1,61 @@ +Jinja2-2.11.2.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4 +Jinja2-2.11.2.dist-info/LICENSE.rst,sha256=O0nc7kEF6ze6wQ-vG-JgQI_oXSUrjp3y4JefweCUQ3s,1475 +Jinja2-2.11.2.dist-info/METADATA,sha256=5ZHRZoIRAMHsJPnqhlJ622_dRPsYePYJ-9EH4-Ry7yI,3535 +Jinja2-2.11.2.dist-info/RECORD,, +Jinja2-2.11.2.dist-info/WHEEL,sha256=kGT74LWyRUZrL4VgLh6_g12IeVl_9u9ZVhadrgXZUEY,110 +Jinja2-2.11.2.dist-info/entry_points.txt,sha256=Qy_DkVo6Xj_zzOtmErrATe8lHZhOqdjpt3e4JJAGyi8,61 +Jinja2-2.11.2.dist-info/top_level.txt,sha256=PkeVWtLb3-CqjWi1fO29OCbj55EhX_chhKrCdrVe_zs,7 +jinja2/__init__.py,sha256=0QCM_jKKDM10yzSdHRVV4mQbCbDqf0GN0GirAqibn9Y,1549 +jinja2/__pycache__/__init__.cpython-38.pyc,, +jinja2/__pycache__/_compat.cpython-38.pyc,, +jinja2/__pycache__/_identifier.cpython-38.pyc,, +jinja2/__pycache__/asyncfilters.cpython-38.pyc,, +jinja2/__pycache__/asyncsupport.cpython-38.pyc,, +jinja2/__pycache__/bccache.cpython-38.pyc,, +jinja2/__pycache__/compiler.cpython-38.pyc,, +jinja2/__pycache__/constants.cpython-38.pyc,, +jinja2/__pycache__/debug.cpython-38.pyc,, +jinja2/__pycache__/defaults.cpython-38.pyc,, +jinja2/__pycache__/environment.cpython-38.pyc,, +jinja2/__pycache__/exceptions.cpython-38.pyc,, +jinja2/__pycache__/ext.cpython-38.pyc,, +jinja2/__pycache__/filters.cpython-38.pyc,, +jinja2/__pycache__/idtracking.cpython-38.pyc,, +jinja2/__pycache__/lexer.cpython-38.pyc,, +jinja2/__pycache__/loaders.cpython-38.pyc,, +jinja2/__pycache__/meta.cpython-38.pyc,, +jinja2/__pycache__/nativetypes.cpython-38.pyc,, +jinja2/__pycache__/nodes.cpython-38.pyc,, +jinja2/__pycache__/optimizer.cpython-38.pyc,, +jinja2/__pycache__/parser.cpython-38.pyc,, +jinja2/__pycache__/runtime.cpython-38.pyc,, +jinja2/__pycache__/sandbox.cpython-38.pyc,, +jinja2/__pycache__/tests.cpython-38.pyc,, +jinja2/__pycache__/utils.cpython-38.pyc,, +jinja2/__pycache__/visitor.cpython-38.pyc,, +jinja2/_compat.py,sha256=B6Se8HjnXVpzz9-vfHejn-DV2NjaVK-Iewupc5kKlu8,3191 +jinja2/_identifier.py,sha256=EdgGJKi7O1yvr4yFlvqPNEqV6M1qHyQr8Gt8GmVTKVM,1775 +jinja2/asyncfilters.py,sha256=XJtYXTxFvcJ5xwk6SaDL4S0oNnT0wPYvXBCSzc482fI,4250 +jinja2/asyncsupport.py,sha256=ZBFsDLuq3Gtji3Ia87lcyuDbqaHZJRdtShZcqwpFnSQ,7209 +jinja2/bccache.py,sha256=3Pmp4jo65M9FQuIxdxoDBbEDFwe4acDMQf77nEJfrHA,12139 +jinja2/compiler.py,sha256=Ta9W1Lit542wItAHXlDcg0sEOsFDMirCdlFPHAurg4o,66284 +jinja2/constants.py,sha256=RR1sTzNzUmKco6aZicw4JpQpJGCuPuqm1h1YmCNUEFY,1458 +jinja2/debug.py,sha256=neR7GIGGjZH3_ILJGVUYy3eLQCCaWJMXOb7o0kGInWc,8529 +jinja2/defaults.py,sha256=85B6YUUCyWPSdrSeVhcqFVuu_bHUAQXeey--FIwSeVQ,1126 +jinja2/environment.py,sha256=XDSLKc4SqNLMOwTSq3TbWEyA5WyXfuLuVD0wAVjEFwM,50629 +jinja2/exceptions.py,sha256=VjNLawcmf2ODffqVMCQK1cRmvFaUfQWF4u8ouP3QPcE,5425 +jinja2/ext.py,sha256=AtwL5O5enT_L3HR9-oBvhGyUTdGoyaqG_ICtnR_EVd4,26441 +jinja2/filters.py,sha256=_RpPgAlgIj7ExvyDzcHAC3B36cocfWK-1TEketbNeM0,41415 +jinja2/idtracking.py,sha256=J3O4VHsrbf3wzwiBc7Cro26kHb6_5kbULeIOzocchIU,9211 +jinja2/lexer.py,sha256=nUFLRKhhKmmEWkLI65nQePgcQs7qsRdjVYZETMt_v0g,30331 +jinja2/loaders.py,sha256=C-fST_dmFjgWkp0ZuCkrgICAoOsoSIF28wfAFink0oU,17666 +jinja2/meta.py,sha256=QjyYhfNRD3QCXjBJpiPl9KgkEkGXJbAkCUq4-Ur10EQ,4131 +jinja2/nativetypes.py,sha256=Ul__gtVw4xH-0qvUvnCNHedQeNDwmEuyLJztzzSPeRg,2753 +jinja2/nodes.py,sha256=Mk1oJPVgIjnQw9WOqILvcu3rLepcFZ0ahxQm2mbwDwc,31095 +jinja2/optimizer.py,sha256=gQLlMYzvQhluhzmAIFA1tXS0cwgWYOjprN-gTRcHVsc,1457 +jinja2/parser.py,sha256=fcfdqePNTNyvosIvczbytVA332qpsURvYnCGcjDHSkA,35660 +jinja2/runtime.py,sha256=0y-BRyIEZ9ltByL2Id6GpHe1oDRQAwNeQvI0SKobNMw,30618 +jinja2/sandbox.py,sha256=knayyUvXsZ-F0mk15mO2-ehK9gsw04UhB8td-iUOtLc,17127 +jinja2/tests.py,sha256=iO_Y-9Vo60zrVe1lMpSl5sKHqAxe2leZHC08OoZ8K24,4799 +jinja2/utils.py,sha256=OoVMlQe9S2-lWT6jJbTu9tDuDvGNyWUhHDcE51i5_Do,22522 +jinja2/visitor.py,sha256=DUHupl0a4PGp7nxRtZFttUzAi1ccxzqc2hzetPYUz8U,3240 diff --git a/magma/squid_cnf/charms/squid/venv/Jinja2-2.11.2.dist-info/WHEEL b/magma/squid_cnf/charms/squid/venv/Jinja2-2.11.2.dist-info/WHEEL new file mode 100644 index 0000000000000000000000000000000000000000..ef99c6cf3283b50a273ac4c6d009a0aa85597070 --- /dev/null +++ b/magma/squid_cnf/charms/squid/venv/Jinja2-2.11.2.dist-info/WHEEL @@ -0,0 +1,6 @@ +Wheel-Version: 1.0 +Generator: bdist_wheel (0.34.2) +Root-Is-Purelib: true +Tag: py2-none-any +Tag: py3-none-any + diff --git a/magma/squid_cnf/charms/squid/venv/Jinja2-2.11.2.dist-info/entry_points.txt b/magma/squid_cnf/charms/squid/venv/Jinja2-2.11.2.dist-info/entry_points.txt new file mode 100644 index 0000000000000000000000000000000000000000..3619483fd4fca7407f3bd462aefcd70d1de69737 --- /dev/null +++ b/magma/squid_cnf/charms/squid/venv/Jinja2-2.11.2.dist-info/entry_points.txt @@ -0,0 +1,3 @@ +[babel.extractors] +jinja2 = jinja2.ext:babel_extract [i18n] + diff --git a/magma/squid_cnf/charms/squid/venv/Jinja2-2.11.2.dist-info/top_level.txt b/magma/squid_cnf/charms/squid/venv/Jinja2-2.11.2.dist-info/top_level.txt new file mode 100644 index 0000000000000000000000000000000000000000..7f7afbf3bf54b346092be6a72070fcbd305ead1e --- /dev/null +++ b/magma/squid_cnf/charms/squid/venv/Jinja2-2.11.2.dist-info/top_level.txt @@ -0,0 +1 @@ +jinja2 diff --git a/magma/squid_cnf/charms/squid/venv/MarkupSafe-1.1.1.dist-info/INSTALLER b/magma/squid_cnf/charms/squid/venv/MarkupSafe-1.1.1.dist-info/INSTALLER new file mode 100644 index 0000000000000000000000000000000000000000..a1b589e38a32041e49332e5e81c2d363dc418d68 --- /dev/null +++ b/magma/squid_cnf/charms/squid/venv/MarkupSafe-1.1.1.dist-info/INSTALLER @@ -0,0 +1 @@ +pip diff --git a/magma/squid_cnf/charms/squid/venv/MarkupSafe-1.1.1.dist-info/LICENSE.txt b/magma/squid_cnf/charms/squid/venv/MarkupSafe-1.1.1.dist-info/LICENSE.txt new file mode 100644 index 0000000000000000000000000000000000000000..9d227a0cc43c3268d15722b763bd94ad298645a1 --- /dev/null +++ b/magma/squid_cnf/charms/squid/venv/MarkupSafe-1.1.1.dist-info/LICENSE.txt @@ -0,0 +1,28 @@ +Copyright 2010 Pallets + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A +PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED +TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/magma/squid_cnf/charms/squid/venv/MarkupSafe-1.1.1.dist-info/METADATA b/magma/squid_cnf/charms/squid/venv/MarkupSafe-1.1.1.dist-info/METADATA new file mode 100644 index 0000000000000000000000000000000000000000..c50370d6e8457be099ff654efbacd8d89d1e9d45 --- /dev/null +++ b/magma/squid_cnf/charms/squid/venv/MarkupSafe-1.1.1.dist-info/METADATA @@ -0,0 +1,105 @@ +Metadata-Version: 2.1 +Name: MarkupSafe +Version: 1.1.1 +Summary: Safely add untrusted strings to HTML/XML markup. +Home-page: https://palletsprojects.com/p/markupsafe/ +Author: Armin Ronacher +Author-email: armin.ronacher@active-4.com +Maintainer: The Pallets Team +Maintainer-email: contact@palletsprojects.com +License: BSD-3-Clause +Project-URL: Documentation, https://markupsafe.palletsprojects.com/ +Project-URL: Code, https://github.com/pallets/markupsafe +Project-URL: Issue tracker, https://github.com/pallets/markupsafe/issues +Platform: UNKNOWN +Classifier: Development Status :: 5 - Production/Stable +Classifier: Environment :: Web Environment +Classifier: Intended Audience :: Developers +Classifier: License :: OSI Approved :: BSD License +Classifier: Operating System :: OS Independent +Classifier: Programming Language :: Python +Classifier: Programming Language :: Python :: 2 +Classifier: Programming Language :: Python :: 2.7 +Classifier: Programming Language :: Python :: 3 +Classifier: Programming Language :: Python :: 3.4 +Classifier: Programming Language :: Python :: 3.5 +Classifier: Programming Language :: Python :: 3.6 +Classifier: Programming Language :: Python :: 3.7 +Classifier: Topic :: Internet :: WWW/HTTP :: Dynamic Content +Classifier: Topic :: Software Development :: Libraries :: Python Modules +Classifier: Topic :: Text Processing :: Markup :: HTML +Requires-Python: >=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.* +Description-Content-Type: text/x-rst + +MarkupSafe +========== + +MarkupSafe implements a text object that escapes characters so it is +safe to use in HTML and XML. Characters that have special meanings are +replaced so that they display as the actual characters. This mitigates +injection attacks, meaning untrusted user input can safely be displayed +on a page. + + +Installing +---------- + +Install and update using `pip`_: + +.. code-block:: text + + pip install -U MarkupSafe + +.. _pip: https://pip.pypa.io/en/stable/quickstart/ + + +Examples +-------- + +.. code-block:: pycon + + >>> from markupsafe import Markup, escape + >>> # escape replaces special characters and wraps in Markup + >>> escape('') + Markup(u'<script>alert(document.cookie);</script>') + >>> # wrap in Markup to mark text "safe" and prevent escaping + >>> Markup('Hello') + Markup('hello') + >>> escape(Markup('Hello')) + Markup('hello') + >>> # Markup is a text subclass (str on Python 3, unicode on Python 2) + >>> # methods and operators escape their arguments + >>> template = Markup("Hello %s") + >>> template % '"World"' + Markup('Hello "World"') + + +Donate +------ + +The Pallets organization develops and supports MarkupSafe and other +libraries that use it. In order to grow the community of contributors +and users, and allow the maintainers to devote more time to the +projects, `please donate today`_. + +.. _please donate today: https://palletsprojects.com/donate + + +Links +----- + +* Website: https://palletsprojects.com/p/markupsafe/ +* Documentation: https://markupsafe.palletsprojects.com/ +* License: `BSD-3-Clause `_ +* Releases: https://pypi.org/project/MarkupSafe/ +* Code: https://github.com/pallets/markupsafe +* Issue tracker: https://github.com/pallets/markupsafe/issues +* Test status: + + * Linux, Mac: https://travis-ci.org/pallets/markupsafe + * Windows: https://ci.appveyor.com/project/pallets/markupsafe + +* Test coverage: https://codecov.io/gh/pallets/markupsafe +* Official chat: https://discord.gg/t6rrQZH + + diff --git a/magma/squid_cnf/charms/squid/venv/MarkupSafe-1.1.1.dist-info/RECORD b/magma/squid_cnf/charms/squid/venv/MarkupSafe-1.1.1.dist-info/RECORD new file mode 100644 index 0000000000000000000000000000000000000000..95c3339900402816081ecce4451852f261c9f088 --- /dev/null +++ b/magma/squid_cnf/charms/squid/venv/MarkupSafe-1.1.1.dist-info/RECORD @@ -0,0 +1,16 @@ +MarkupSafe-1.1.1.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4 +MarkupSafe-1.1.1.dist-info/LICENSE.txt,sha256=SJqOEQhQntmKN7uYPhHg9-HTHwvY-Zp5yESOf_N9B-o,1475 +MarkupSafe-1.1.1.dist-info/METADATA,sha256=IFCP4hCNGjXJgMoSvdjPiKDLAMUTTWoxKXQsQvmyMNU,3653 +MarkupSafe-1.1.1.dist-info/RECORD,, +MarkupSafe-1.1.1.dist-info/WHEEL,sha256=VEyGcIFAmk_1KbI6gaZGw_mMiT-pdGweASQLX-DzYaY,108 +MarkupSafe-1.1.1.dist-info/top_level.txt,sha256=qy0Plje5IJuvsCBjejJyhDCjEAdcDLK_2agVcex8Z6U,11 +markupsafe/__init__.py,sha256=oTblO5f9KFM-pvnq9bB0HgElnqkJyqHnFN1Nx2NIvnY,10126 +markupsafe/__pycache__/__init__.cpython-38.pyc,, +markupsafe/__pycache__/_compat.cpython-38.pyc,, +markupsafe/__pycache__/_constants.cpython-38.pyc,, +markupsafe/__pycache__/_native.cpython-38.pyc,, +markupsafe/_compat.py,sha256=uEW1ybxEjfxIiuTbRRaJpHsPFf4yQUMMKaPgYEC5XbU,558 +markupsafe/_constants.py,sha256=zo2ajfScG-l1Sb_52EP3MlDCqO7Y1BVHUXXKRsVDRNk,4690 +markupsafe/_native.py,sha256=d-8S_zzYt2y512xYcuSxq0NeG2DUUvG80wVdTn-4KI8,1873 +markupsafe/_speedups.c,sha256=k0fzEIK3CP6MmMqeY0ob43TP90mVN0DTyn7BAl3RqSg,9884 +markupsafe/_speedups.cpython-38-x86_64-linux-gnu.so,sha256=SbJwN321Xn7OPYGv5a6Ghzga75uT8RHQUGkoQUASF-o,48016 diff --git a/magma/squid_cnf/charms/squid/venv/MarkupSafe-1.1.1.dist-info/WHEEL b/magma/squid_cnf/charms/squid/venv/MarkupSafe-1.1.1.dist-info/WHEEL new file mode 100644 index 0000000000000000000000000000000000000000..ae40efdd2fb4358a9bd6a35167512e7e822f7f4b --- /dev/null +++ b/magma/squid_cnf/charms/squid/venv/MarkupSafe-1.1.1.dist-info/WHEEL @@ -0,0 +1,5 @@ +Wheel-Version: 1.0 +Generator: bdist_wheel (0.31.1) +Root-Is-Purelib: false +Tag: cp38-cp38-manylinux1_x86_64 + diff --git a/magma/squid_cnf/charms/squid/venv/MarkupSafe-1.1.1.dist-info/top_level.txt b/magma/squid_cnf/charms/squid/venv/MarkupSafe-1.1.1.dist-info/top_level.txt new file mode 100644 index 0000000000000000000000000000000000000000..75bf729258f9daef77370b6df1a57940f90fc23f --- /dev/null +++ b/magma/squid_cnf/charms/squid/venv/MarkupSafe-1.1.1.dist-info/top_level.txt @@ -0,0 +1 @@ +markupsafe diff --git a/magma/squid_cnf/charms/squid/venv/PyYAML-5.3.1.dist-info/INSTALLER b/magma/squid_cnf/charms/squid/venv/PyYAML-5.3.1.dist-info/INSTALLER new file mode 100644 index 0000000000000000000000000000000000000000..a1b589e38a32041e49332e5e81c2d363dc418d68 --- /dev/null +++ b/magma/squid_cnf/charms/squid/venv/PyYAML-5.3.1.dist-info/INSTALLER @@ -0,0 +1 @@ +pip diff --git a/magma/squid_cnf/charms/squid/venv/PyYAML-5.3.1.dist-info/LICENSE b/magma/squid_cnf/charms/squid/venv/PyYAML-5.3.1.dist-info/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..3d82c281ee8ceb45368153139f5935bd8d39c816 --- /dev/null +++ b/magma/squid_cnf/charms/squid/venv/PyYAML-5.3.1.dist-info/LICENSE @@ -0,0 +1,20 @@ +Copyright (c) 2017-2020 Ingy döt Net +Copyright (c) 2006-2016 Kirill Simonov + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/magma/squid_cnf/charms/squid/venv/PyYAML-5.3.1.dist-info/METADATA b/magma/squid_cnf/charms/squid/venv/PyYAML-5.3.1.dist-info/METADATA new file mode 100644 index 0000000000000000000000000000000000000000..a70dd202d485af74ab22294dfbc7710364d0b127 --- /dev/null +++ b/magma/squid_cnf/charms/squid/venv/PyYAML-5.3.1.dist-info/METADATA @@ -0,0 +1,41 @@ +Metadata-Version: 2.1 +Name: PyYAML +Version: 5.3.1 +Summary: YAML parser and emitter for Python +Home-page: https://github.com/yaml/pyyaml +Author: Kirill Simonov +Author-email: xi@resolvent.net +License: MIT +Download-URL: https://pypi.org/project/PyYAML/ +Platform: Any +Classifier: Development Status :: 5 - Production/Stable +Classifier: Intended Audience :: Developers +Classifier: License :: OSI Approved :: MIT License +Classifier: Operating System :: OS Independent +Classifier: Programming Language :: Cython +Classifier: Programming Language :: Python +Classifier: Programming Language :: Python :: 2 +Classifier: Programming Language :: Python :: 2.7 +Classifier: Programming Language :: Python :: 3 +Classifier: Programming Language :: Python :: 3.5 +Classifier: Programming Language :: Python :: 3.6 +Classifier: Programming Language :: Python :: 3.7 +Classifier: Programming Language :: Python :: 3.8 +Classifier: Programming Language :: Python :: Implementation :: CPython +Classifier: Programming Language :: Python :: Implementation :: PyPy +Classifier: Topic :: Software Development :: Libraries :: Python Modules +Classifier: Topic :: Text Processing :: Markup +Requires-Python: >=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.* + +YAML is a data serialization format designed for human readability +and interaction with scripting languages. PyYAML is a YAML parser +and emitter for Python. + +PyYAML features a complete YAML 1.1 parser, Unicode support, pickle +support, capable extension API, and sensible error messages. PyYAML +supports standard YAML tags and provides Python-specific tags that +allow to represent an arbitrary Python object. + +PyYAML is applicable for a broad range of tasks from complex +configuration files to object serialization and persistence. + diff --git a/magma/squid_cnf/charms/squid/venv/PyYAML-5.3.1.dist-info/RECORD b/magma/squid_cnf/charms/squid/venv/PyYAML-5.3.1.dist-info/RECORD new file mode 100644 index 0000000000000000000000000000000000000000..cae61d9785c50e2d66319fde9ccc891f7f960354 --- /dev/null +++ b/magma/squid_cnf/charms/squid/venv/PyYAML-5.3.1.dist-info/RECORD @@ -0,0 +1,40 @@ +PyYAML-5.3.1.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4 +PyYAML-5.3.1.dist-info/LICENSE,sha256=xAESRJ8lS5dTBFklJIMT6ScO-jbSJrItgtTMbEPFfyk,1101 +PyYAML-5.3.1.dist-info/METADATA,sha256=xTsZFjd8T4M-5rC2M3BHgx_KTTpEPy5vFDIXrbzRXPQ,1758 +PyYAML-5.3.1.dist-info/RECORD,, +PyYAML-5.3.1.dist-info/WHEEL,sha256=TpFVeXF_cAlV118WSIPWtjqW7nPvzoOw-49FmS3fDKQ,103 +PyYAML-5.3.1.dist-info/top_level.txt,sha256=rpj0IVMTisAjh_1vG3Ccf9v5jpCQwAz6cD1IVU5ZdhQ,11 +yaml/__init__.py,sha256=XFUNbKTg4afAd0BETjGQ1mKQ97_g5jbE1C0WoKc74dc,13170 +yaml/__pycache__/__init__.cpython-38.pyc,, +yaml/__pycache__/composer.cpython-38.pyc,, +yaml/__pycache__/constructor.cpython-38.pyc,, +yaml/__pycache__/cyaml.cpython-38.pyc,, +yaml/__pycache__/dumper.cpython-38.pyc,, +yaml/__pycache__/emitter.cpython-38.pyc,, +yaml/__pycache__/error.cpython-38.pyc,, +yaml/__pycache__/events.cpython-38.pyc,, +yaml/__pycache__/loader.cpython-38.pyc,, +yaml/__pycache__/nodes.cpython-38.pyc,, +yaml/__pycache__/parser.cpython-38.pyc,, +yaml/__pycache__/reader.cpython-38.pyc,, +yaml/__pycache__/representer.cpython-38.pyc,, +yaml/__pycache__/resolver.cpython-38.pyc,, +yaml/__pycache__/scanner.cpython-38.pyc,, +yaml/__pycache__/serializer.cpython-38.pyc,, +yaml/__pycache__/tokens.cpython-38.pyc,, +yaml/composer.py,sha256=_Ko30Wr6eDWUeUpauUGT3Lcg9QPBnOPVlTnIMRGJ9FM,4883 +yaml/constructor.py,sha256=O3Uaf0_J_5GQBoeI9ZNhpJAhtdagr_X2HzDgGbZOMnw,28627 +yaml/cyaml.py,sha256=LiMkvchNonfoy1F6ec9L2BiUz3r0bwF4hympASJX1Ic,3846 +yaml/dumper.py,sha256=PLctZlYwZLp7XmeUdwRuv4nYOZ2UBnDIUy8-lKfLF-o,2837 +yaml/emitter.py,sha256=jghtaU7eFwg31bG0B7RZea_29Adi9CKmXq_QjgQpCkQ,43006 +yaml/error.py,sha256=Ah9z-toHJUbE9j-M8YpxgSRM5CgLCcwVzJgLLRF2Fxo,2533 +yaml/events.py,sha256=50_TksgQiE4up-lKo_V-nBy-tAIxkIPQxY5qDhKCeHw,2445 +yaml/loader.py,sha256=UVa-zIqmkFSCIYq_PgSGm4NSJttHY2Rf_zQ4_b1fHN0,2061 +yaml/nodes.py,sha256=gPKNj8pKCdh2d4gr3gIYINnPOaOxGhJAUiYhGRnPE84,1440 +yaml/parser.py,sha256=ilWp5vvgoHFGzvOZDItFoGjD6D42nhlZrZyjAwa0oJo,25495 +yaml/reader.py,sha256=0dmzirOiDG4Xo41RnuQS7K9rkY3xjHiVasfDMNTqCNw,6794 +yaml/representer.py,sha256=82UM3ZxUQKqsKAF4ltWOxCS6jGPIFtXpGs7mvqyv4Xs,14184 +yaml/resolver.py,sha256=DJCjpQr8YQCEYYjKEYqTl0GrsZil2H4aFOI9b0Oe-U4,8970 +yaml/scanner.py,sha256=KeQIKGNlSyPE8QDwionHxy9CgbqE5teJEz05FR9-nAg,51277 +yaml/serializer.py,sha256=ChuFgmhU01hj4xgI8GaKv6vfM2Bujwa9i7d2FAHj7cA,4165 +yaml/tokens.py,sha256=lTQIzSVw8Mg9wv459-TjiOQe6wVziqaRlqX2_89rp54,2573 diff --git a/magma/squid_cnf/charms/squid/venv/PyYAML-5.3.1.dist-info/WHEEL b/magma/squid_cnf/charms/squid/venv/PyYAML-5.3.1.dist-info/WHEEL new file mode 100644 index 0000000000000000000000000000000000000000..d193dea988d97c2f7f7bf3c4fc196496d361cd4d --- /dev/null +++ b/magma/squid_cnf/charms/squid/venv/PyYAML-5.3.1.dist-info/WHEEL @@ -0,0 +1,5 @@ +Wheel-Version: 1.0 +Generator: bdist_wheel (0.34.2) +Root-Is-Purelib: false +Tag: cp38-cp38-linux_x86_64 + diff --git a/magma/squid_cnf/charms/squid/venv/PyYAML-5.3.1.dist-info/top_level.txt b/magma/squid_cnf/charms/squid/venv/PyYAML-5.3.1.dist-info/top_level.txt new file mode 100644 index 0000000000000000000000000000000000000000..e6475e911f628412049bc4090d86f23ac403adde --- /dev/null +++ b/magma/squid_cnf/charms/squid/venv/PyYAML-5.3.1.dist-info/top_level.txt @@ -0,0 +1,2 @@ +_yaml +yaml diff --git a/magma/squid_cnf/charms/squid/venv/jinja2/__init__.py b/magma/squid_cnf/charms/squid/venv/jinja2/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..1229ba4275a0549e01dd6e7b10e1ccf17f17389f --- /dev/null +++ b/magma/squid_cnf/charms/squid/venv/jinja2/__init__.py @@ -0,0 +1,44 @@ +# -*- coding: utf-8 -*- +"""Jinja is a template engine written in pure Python. It provides a +non-XML syntax that supports inline expressions and an optional +sandboxed environment. +""" +from markupsafe import escape +from markupsafe import Markup + +from .bccache import BytecodeCache +from .bccache import FileSystemBytecodeCache +from .bccache import MemcachedBytecodeCache +from .environment import Environment +from .environment import Template +from .exceptions import TemplateAssertionError +from .exceptions import TemplateError +from .exceptions import TemplateNotFound +from .exceptions import TemplateRuntimeError +from .exceptions import TemplatesNotFound +from .exceptions import TemplateSyntaxError +from .exceptions import UndefinedError +from .filters import contextfilter +from .filters import environmentfilter +from .filters import evalcontextfilter +from .loaders import BaseLoader +from .loaders import ChoiceLoader +from .loaders import DictLoader +from .loaders import FileSystemLoader +from .loaders import FunctionLoader +from .loaders import ModuleLoader +from .loaders import PackageLoader +from .loaders import PrefixLoader +from .runtime import ChainableUndefined +from .runtime import DebugUndefined +from .runtime import make_logging_undefined +from .runtime import StrictUndefined +from .runtime import Undefined +from .utils import clear_caches +from .utils import contextfunction +from .utils import environmentfunction +from .utils import evalcontextfunction +from .utils import is_undefined +from .utils import select_autoescape + +__version__ = "2.11.2" diff --git a/magma/squid_cnf/charms/squid/venv/jinja2/_compat.py b/magma/squid_cnf/charms/squid/venv/jinja2/_compat.py new file mode 100644 index 0000000000000000000000000000000000000000..1f044954a02933bcec2277fcdd575821bc18a99a --- /dev/null +++ b/magma/squid_cnf/charms/squid/venv/jinja2/_compat.py @@ -0,0 +1,132 @@ +# -*- coding: utf-8 -*- +# flake8: noqa +import marshal +import sys + +PY2 = sys.version_info[0] == 2 +PYPY = hasattr(sys, "pypy_translation_info") +_identity = lambda x: x + +if not PY2: + unichr = chr + range_type = range + text_type = str + string_types = (str,) + integer_types = (int,) + + iterkeys = lambda d: iter(d.keys()) + itervalues = lambda d: iter(d.values()) + iteritems = lambda d: iter(d.items()) + + import pickle + from io import BytesIO, StringIO + + NativeStringIO = StringIO + + def reraise(tp, value, tb=None): + if value.__traceback__ is not tb: + raise value.with_traceback(tb) + raise value + + ifilter = filter + imap = map + izip = zip + intern = sys.intern + + implements_iterator = _identity + implements_to_string = _identity + encode_filename = _identity + + marshal_dump = marshal.dump + marshal_load = marshal.load + +else: + unichr = unichr + text_type = unicode + range_type = xrange + string_types = (str, unicode) + integer_types = (int, long) + + iterkeys = lambda d: d.iterkeys() + itervalues = lambda d: d.itervalues() + iteritems = lambda d: d.iteritems() + + import cPickle as pickle + from cStringIO import StringIO as BytesIO, StringIO + + NativeStringIO = BytesIO + + exec("def reraise(tp, value, tb=None):\n raise tp, value, tb") + + from itertools import imap, izip, ifilter + + intern = intern + + def implements_iterator(cls): + cls.next = cls.__next__ + del cls.__next__ + return cls + + def implements_to_string(cls): + cls.__unicode__ = cls.__str__ + cls.__str__ = lambda x: x.__unicode__().encode("utf-8") + return cls + + def encode_filename(filename): + if isinstance(filename, unicode): + return filename.encode("utf-8") + return filename + + def marshal_dump(code, f): + if isinstance(f, file): + marshal.dump(code, f) + else: + f.write(marshal.dumps(code)) + + def marshal_load(f): + if isinstance(f, file): + return marshal.load(f) + return marshal.loads(f.read()) + + +def with_metaclass(meta, *bases): + """Create a base class with a metaclass.""" + # This requires a bit of explanation: the basic idea is to make a + # dummy metaclass for one level of class instantiation that replaces + # itself with the actual metaclass. + class metaclass(type): + def __new__(cls, name, this_bases, d): + return meta(name, bases, d) + + return type.__new__(metaclass, "temporary_class", (), {}) + + +try: + from urllib.parse import quote_from_bytes as url_quote +except ImportError: + from urllib import quote as url_quote + + +try: + from collections import abc +except ImportError: + import collections as abc + + +try: + from os import fspath +except ImportError: + try: + from pathlib import PurePath + except ImportError: + PurePath = None + + def fspath(path): + if hasattr(path, "__fspath__"): + return path.__fspath__() + + # Python 3.5 doesn't have __fspath__ yet, use str. + if PurePath is not None and isinstance(path, PurePath): + return str(path) + + return path diff --git a/magma/squid_cnf/charms/squid/venv/jinja2/_identifier.py b/magma/squid_cnf/charms/squid/venv/jinja2/_identifier.py new file mode 100644 index 0000000000000000000000000000000000000000..224d5449d138e75f4f3c25d70cb8f1ea54ccd047 --- /dev/null +++ b/magma/squid_cnf/charms/squid/venv/jinja2/_identifier.py @@ -0,0 +1,6 @@ +import re + +# generated by scripts/generate_identifier_pattern.py +pattern = re.compile( + r"[\w·̀-ͯ·҃-֑҇-ׇֽֿׁׂׅׄؐ-ًؚ-ٰٟۖ-ۜ۟-۪ۤۧۨ-ܑۭܰ-݊ަ-ް߫-߳ࠖ-࠙ࠛ-ࠣࠥ-ࠧࠩ-࡙࠭-࡛ࣔ-ࣣ࣡-ःऺ-़ा-ॏ॑-ॗॢॣঁ-ঃ়া-ৄেৈো-্ৗৢৣਁ-ਃ਼ਾ-ੂੇੈੋ-੍ੑੰੱੵઁ-ઃ઼ા-ૅે-ૉો-્ૢૣଁ-ଃ଼ା-ୄେୈୋ-୍ୖୗୢୣஂா-ூெ-ைொ-்ௗఀ-ఃా-ౄె-ైొ-్ౕౖౢౣಁ-ಃ಼ಾ-ೄೆ-ೈೊ-್ೕೖೢೣഁ-ഃാ-ൄെ-ൈൊ-്ൗൢൣංඃ්ා-ුූෘ-ෟෲෳัิ-ฺ็-๎ັິ-ູົຼ່-ໍ༹༘༙༵༷༾༿ཱ-྄྆྇ྍ-ྗྙ-ྼ࿆ါ-ှၖ-ၙၞ-ၠၢ-ၤၧ-ၭၱ-ၴႂ-ႍႏႚ-ႝ፝-፟ᜒ-᜔ᜲ-᜴ᝒᝓᝲᝳ឴-៓៝᠋-᠍ᢅᢆᢩᤠ-ᤫᤰ-᤻ᨗ-ᨛᩕ-ᩞ᩠-᩿᩼᪰-᪽ᬀ-ᬄ᬴-᭄᭫-᭳ᮀ-ᮂᮡ-ᮭ᯦-᯳ᰤ-᰷᳐-᳔᳒-᳨᳭ᳲ-᳴᳸᳹᷀-᷵᷻-᷿‿⁀⁔⃐-⃥⃜⃡-⃰℘℮⳯-⵿⳱ⷠ-〪ⷿ-゙゚〯꙯ꙴ-꙽ꚞꚟ꛰꛱ꠂ꠆ꠋꠣ-ꠧꢀꢁꢴ-ꣅ꣠-꣱ꤦ-꤭ꥇ-꥓ꦀ-ꦃ꦳-꧀ꧥꨩ-ꨶꩃꩌꩍꩻ-ꩽꪰꪲ-ꪴꪷꪸꪾ꪿꫁ꫫ-ꫯꫵ꫶ꯣ-ꯪ꯬꯭ﬞ︀-️︠-︯︳︴﹍-﹏＿𐇽𐋠𐍶-𐍺𐨁-𐨃𐨅𐨆𐨌-𐨏𐨸-𐨿𐨺𐫦𐫥𑀀-𑀂𑀸-𑁆𑁿-𑂂𑂰-𑂺𑄀-𑄂𑄧-𑅳𑄴𑆀-𑆂𑆳-𑇊𑇀-𑇌𑈬-𑈷𑈾𑋟-𑋪𑌀-𑌃𑌼𑌾-𑍄𑍇𑍈𑍋-𑍍𑍗𑍢𑍣𑍦-𑍬𑍰-𑍴𑐵-𑑆𑒰-𑓃𑖯-𑖵𑖸-𑗀𑗜𑗝𑘰-𑙀𑚫-𑚷𑜝-𑜫𑰯-𑰶𑰸-𑰿𑲒-𑲧𑲩-𑲶𖫰-𖫴𖬰-𖬶𖽑-𖽾𖾏-𖾒𛲝𛲞𝅥-𝅩𝅭-𝅲𝅻-𝆂𝆅-𝆋𝆪-𝆭𝉂-𝉄𝨀-𝨶𝨻-𝩬𝩵𝪄𝪛-𝪟𝪡-𝪯𞀀-𞀆𞀈-𞀘𞀛-𞀡𞀣𞀤𞀦-𞣐𞀪-𞣖𞥄-𞥊󠄀-󠇯]+" # noqa: B950 +) diff --git a/magma/squid_cnf/charms/squid/venv/jinja2/asyncfilters.py b/magma/squid_cnf/charms/squid/venv/jinja2/asyncfilters.py new file mode 100644 index 0000000000000000000000000000000000000000..3d98dbcc00de104a584b0707df8f0d4e8bbe1376 --- /dev/null +++ b/magma/squid_cnf/charms/squid/venv/jinja2/asyncfilters.py @@ -0,0 +1,158 @@ +from functools import wraps + +from . import filters +from .asyncsupport import auto_aiter +from .asyncsupport import auto_await + + +async def auto_to_seq(value): + seq = [] + if hasattr(value, "__aiter__"): + async for item in value: + seq.append(item) + else: + for item in value: + seq.append(item) + return seq + + +async def async_select_or_reject(args, kwargs, modfunc, lookup_attr): + seq, func = filters.prepare_select_or_reject(args, kwargs, modfunc, lookup_attr) + if seq: + async for item in auto_aiter(seq): + if func(item): + yield item + + +def dualfilter(normal_filter, async_filter): + wrap_evalctx = False + if getattr(normal_filter, "environmentfilter", False) is True: + + def is_async(args): + return args[0].is_async + + wrap_evalctx = False + else: + has_evalctxfilter = getattr(normal_filter, "evalcontextfilter", False) is True + has_ctxfilter = getattr(normal_filter, "contextfilter", False) is True + wrap_evalctx = not has_evalctxfilter and not has_ctxfilter + + def is_async(args): + return args[0].environment.is_async + + @wraps(normal_filter) + def wrapper(*args, **kwargs): + b = is_async(args) + if wrap_evalctx: + args = args[1:] + if b: + return async_filter(*args, **kwargs) + return normal_filter(*args, **kwargs) + + if wrap_evalctx: + wrapper.evalcontextfilter = True + + wrapper.asyncfiltervariant = True + + return wrapper + + +def asyncfiltervariant(original): + def decorator(f): + return dualfilter(original, f) + + return decorator + + +@asyncfiltervariant(filters.do_first) +async def do_first(environment, seq): + try: + return await auto_aiter(seq).__anext__() + except StopAsyncIteration: + return environment.undefined("No first item, sequence was empty.") + + +@asyncfiltervariant(filters.do_groupby) +async def do_groupby(environment, value, attribute): + expr = filters.make_attrgetter(environment, attribute) + return [ + filters._GroupTuple(key, await auto_to_seq(values)) + for key, values in filters.groupby( + sorted(await auto_to_seq(value), key=expr), expr + ) + ] + + +@asyncfiltervariant(filters.do_join) +async def do_join(eval_ctx, value, d=u"", attribute=None): + return filters.do_join(eval_ctx, await auto_to_seq(value), d, attribute) + + +@asyncfiltervariant(filters.do_list) +async def do_list(value): + return await auto_to_seq(value) + + +@asyncfiltervariant(filters.do_reject) +async def do_reject(*args, **kwargs): + return async_select_or_reject(args, kwargs, lambda x: not x, False) + + +@asyncfiltervariant(filters.do_rejectattr) +async def do_rejectattr(*args, **kwargs): + return async_select_or_reject(args, kwargs, lambda x: not x, True) + + +@asyncfiltervariant(filters.do_select) +async def do_select(*args, **kwargs): + return async_select_or_reject(args, kwargs, lambda x: x, False) + + +@asyncfiltervariant(filters.do_selectattr) +async def do_selectattr(*args, **kwargs): + return async_select_or_reject(args, kwargs, lambda x: x, True) + + +@asyncfiltervariant(filters.do_map) +async def do_map(*args, **kwargs): + seq, func = filters.prepare_map(args, kwargs) + if seq: + async for item in auto_aiter(seq): + yield await auto_await(func(item)) + + +@asyncfiltervariant(filters.do_sum) +async def do_sum(environment, iterable, attribute=None, start=0): + rv = start + if attribute is not None: + func = filters.make_attrgetter(environment, attribute) + else: + + def func(x): + return x + + async for item in auto_aiter(iterable): + rv += func(item) + return rv + + +@asyncfiltervariant(filters.do_slice) +async def do_slice(value, slices, fill_with=None): + return filters.do_slice(await auto_to_seq(value), slices, fill_with) + + +ASYNC_FILTERS = { + "first": do_first, + "groupby": do_groupby, + "join": do_join, + "list": do_list, + # we intentionally do not support do_last because that would be + # ridiculous + "reject": do_reject, + "rejectattr": do_rejectattr, + "map": do_map, + "select": do_select, + "selectattr": do_selectattr, + "sum": do_sum, + "slice": do_slice, +} diff --git a/magma/squid_cnf/charms/squid/venv/jinja2/asyncsupport.py b/magma/squid_cnf/charms/squid/venv/jinja2/asyncsupport.py new file mode 100644 index 0000000000000000000000000000000000000000..78ba3739d8dee1e644f96e32f833279d941c3f65 --- /dev/null +++ b/magma/squid_cnf/charms/squid/venv/jinja2/asyncsupport.py @@ -0,0 +1,264 @@ +# -*- coding: utf-8 -*- +"""The code for async support. Importing this patches Jinja on supported +Python versions. +""" +import asyncio +import inspect +from functools import update_wrapper + +from markupsafe import Markup + +from .environment import TemplateModule +from .runtime import LoopContext +from .utils import concat +from .utils import internalcode +from .utils import missing + + +async def concat_async(async_gen): + rv = [] + + async def collect(): + async for event in async_gen: + rv.append(event) + + await collect() + return concat(rv) + + +async def generate_async(self, *args, **kwargs): + vars = dict(*args, **kwargs) + try: + async for event in self.root_render_func(self.new_context(vars)): + yield event + except Exception: + yield self.environment.handle_exception() + + +def wrap_generate_func(original_generate): + def _convert_generator(self, loop, args, kwargs): + async_gen = self.generate_async(*args, **kwargs) + try: + while 1: + yield loop.run_until_complete(async_gen.__anext__()) + except StopAsyncIteration: + pass + + def generate(self, *args, **kwargs): + if not self.environment.is_async: + return original_generate(self, *args, **kwargs) + return _convert_generator(self, asyncio.get_event_loop(), args, kwargs) + + return update_wrapper(generate, original_generate) + + +async def render_async(self, *args, **kwargs): + if not self.environment.is_async: + raise RuntimeError("The environment was not created with async mode enabled.") + + vars = dict(*args, **kwargs) + ctx = self.new_context(vars) + + try: + return await concat_async(self.root_render_func(ctx)) + except Exception: + return self.environment.handle_exception() + + +def wrap_render_func(original_render): + def render(self, *args, **kwargs): + if not self.environment.is_async: + return original_render(self, *args, **kwargs) + loop = asyncio.get_event_loop() + return loop.run_until_complete(self.render_async(*args, **kwargs)) + + return update_wrapper(render, original_render) + + +def wrap_block_reference_call(original_call): + @internalcode + async def async_call(self): + rv = await concat_async(self._stack[self._depth](self._context)) + if self._context.eval_ctx.autoescape: + rv = Markup(rv) + return rv + + @internalcode + def __call__(self): + if not self._context.environment.is_async: + return original_call(self) + return async_call(self) + + return update_wrapper(__call__, original_call) + + +def wrap_macro_invoke(original_invoke): + @internalcode + async def async_invoke(self, arguments, autoescape): + rv = await self._func(*arguments) + if autoescape: + rv = Markup(rv) + return rv + + @internalcode + def _invoke(self, arguments, autoescape): + if not self._environment.is_async: + return original_invoke(self, arguments, autoescape) + return async_invoke(self, arguments, autoescape) + + return update_wrapper(_invoke, original_invoke) + + +@internalcode +async def get_default_module_async(self): + if self._module is not None: + return self._module + self._module = rv = await self.make_module_async() + return rv + + +def wrap_default_module(original_default_module): + @internalcode + def _get_default_module(self): + if self.environment.is_async: + raise RuntimeError("Template module attribute is unavailable in async mode") + return original_default_module(self) + + return _get_default_module + + +async def make_module_async(self, vars=None, shared=False, locals=None): + context = self.new_context(vars, shared, locals) + body_stream = [] + async for item in self.root_render_func(context): + body_stream.append(item) + return TemplateModule(self, context, body_stream) + + +def patch_template(): + from . import Template + + Template.generate = wrap_generate_func(Template.generate) + Template.generate_async = update_wrapper(generate_async, Template.generate_async) + Template.render_async = update_wrapper(render_async, Template.render_async) + Template.render = wrap_render_func(Template.render) + Template._get_default_module = wrap_default_module(Template._get_default_module) + Template._get_default_module_async = get_default_module_async + Template.make_module_async = update_wrapper( + make_module_async, Template.make_module_async + ) + + +def patch_runtime(): + from .runtime import BlockReference, Macro + + BlockReference.__call__ = wrap_block_reference_call(BlockReference.__call__) + Macro._invoke = wrap_macro_invoke(Macro._invoke) + + +def patch_filters(): + from .filters import FILTERS + from .asyncfilters import ASYNC_FILTERS + + FILTERS.update(ASYNC_FILTERS) + + +def patch_all(): + patch_template() + patch_runtime() + patch_filters() + + +async def auto_await(value): + if inspect.isawaitable(value): + return await value + return value + + +async def auto_aiter(iterable): + if hasattr(iterable, "__aiter__"): + async for item in iterable: + yield item + return + for item in iterable: + yield item + + +class AsyncLoopContext(LoopContext): + _to_iterator = staticmethod(auto_aiter) + + @property + async def length(self): + if self._length is not None: + return self._length + + try: + self._length = len(self._iterable) + except TypeError: + iterable = [x async for x in self._iterator] + self._iterator = self._to_iterator(iterable) + self._length = len(iterable) + self.index + (self._after is not missing) + + return self._length + + @property + async def revindex0(self): + return await self.length - self.index + + @property + async def revindex(self): + return await self.length - self.index0 + + async def _peek_next(self): + if self._after is not missing: + return self._after + + try: + self._after = await self._iterator.__anext__() + except StopAsyncIteration: + self._after = missing + + return self._after + + @property + async def last(self): + return await self._peek_next() is missing + + @property + async def nextitem(self): + rv = await self._peek_next() + + if rv is missing: + return self._undefined("there is no next item") + + return rv + + def __aiter__(self): + return self + + async def __anext__(self): + if self._after is not missing: + rv = self._after + self._after = missing + else: + rv = await self._iterator.__anext__() + + self.index0 += 1 + self._before = self._current + self._current = rv + return rv, self + + +async def make_async_loop_context(iterable, undefined, recurse=None, depth0=0): + import warnings + + warnings.warn( + "This template must be recompiled with at least Jinja 2.11, or" + " it will fail in 3.0.", + DeprecationWarning, + stacklevel=2, + ) + return AsyncLoopContext(iterable, undefined, recurse, depth0) + + +patch_all() diff --git a/magma/squid_cnf/charms/squid/venv/jinja2/bccache.py b/magma/squid_cnf/charms/squid/venv/jinja2/bccache.py new file mode 100644 index 0000000000000000000000000000000000000000..9c0661030f7af4b2b186b1726341c910194b7cbd --- /dev/null +++ b/magma/squid_cnf/charms/squid/venv/jinja2/bccache.py @@ -0,0 +1,350 @@ +# -*- coding: utf-8 -*- +"""The optional bytecode cache system. This is useful if you have very +complex template situations and the compilation of all those templates +slows down your application too much. + +Situations where this is useful are often forking web applications that +are initialized on the first request. +""" +import errno +import fnmatch +import os +import stat +import sys +import tempfile +from hashlib import sha1 +from os import listdir +from os import path + +from ._compat import BytesIO +from ._compat import marshal_dump +from ._compat import marshal_load +from ._compat import pickle +from ._compat import text_type +from .utils import open_if_exists + +bc_version = 4 +# Magic bytes to identify Jinja bytecode cache files. Contains the +# Python major and minor version to avoid loading incompatible bytecode +# if a project upgrades its Python version. +bc_magic = ( + b"j2" + + pickle.dumps(bc_version, 2) + + pickle.dumps((sys.version_info[0] << 24) | sys.version_info[1], 2) +) + + +class Bucket(object): + """Buckets are used to store the bytecode for one template. It's created + and initialized by the bytecode cache and passed to the loading functions. + + The buckets get an internal checksum from the cache assigned and use this + to automatically reject outdated cache material. Individual bytecode + cache subclasses don't have to care about cache invalidation. + """ + + def __init__(self, environment, key, checksum): + self.environment = environment + self.key = key + self.checksum = checksum + self.reset() + + def reset(self): + """Resets the bucket (unloads the bytecode).""" + self.code = None + + def load_bytecode(self, f): + """Loads bytecode from a file or file like object.""" + # make sure the magic header is correct + magic = f.read(len(bc_magic)) + if magic != bc_magic: + self.reset() + return + # the source code of the file changed, we need to reload + checksum = pickle.load(f) + if self.checksum != checksum: + self.reset() + return + # if marshal_load fails then we need to reload + try: + self.code = marshal_load(f) + except (EOFError, ValueError, TypeError): + self.reset() + return + + def write_bytecode(self, f): + """Dump the bytecode into the file or file like object passed.""" + if self.code is None: + raise TypeError("can't write empty bucket") + f.write(bc_magic) + pickle.dump(self.checksum, f, 2) + marshal_dump(self.code, f) + + def bytecode_from_string(self, string): + """Load bytecode from a string.""" + self.load_bytecode(BytesIO(string)) + + def bytecode_to_string(self): + """Return the bytecode as string.""" + out = BytesIO() + self.write_bytecode(out) + return out.getvalue() + + +class BytecodeCache(object): + """To implement your own bytecode cache you have to subclass this class + and override :meth:`load_bytecode` and :meth:`dump_bytecode`. Both of + these methods are passed a :class:`~jinja2.bccache.Bucket`. + + A very basic bytecode cache that saves the bytecode on the file system:: + + from os import path + + class MyCache(BytecodeCache): + + def __init__(self, directory): + self.directory = directory + + def load_bytecode(self, bucket): + filename = path.join(self.directory, bucket.key) + if path.exists(filename): + with open(filename, 'rb') as f: + bucket.load_bytecode(f) + + def dump_bytecode(self, bucket): + filename = path.join(self.directory, bucket.key) + with open(filename, 'wb') as f: + bucket.write_bytecode(f) + + A more advanced version of a filesystem based bytecode cache is part of + Jinja. + """ + + def load_bytecode(self, bucket): + """Subclasses have to override this method to load bytecode into a + bucket. If they are not able to find code in the cache for the + bucket, it must not do anything. + """ + raise NotImplementedError() + + def dump_bytecode(self, bucket): + """Subclasses have to override this method to write the bytecode + from a bucket back to the cache. If it unable to do so it must not + fail silently but raise an exception. + """ + raise NotImplementedError() + + def clear(self): + """Clears the cache. This method is not used by Jinja but should be + implemented to allow applications to clear the bytecode cache used + by a particular environment. + """ + + def get_cache_key(self, name, filename=None): + """Returns the unique hash key for this template name.""" + hash = sha1(name.encode("utf-8")) + if filename is not None: + filename = "|" + filename + if isinstance(filename, text_type): + filename = filename.encode("utf-8") + hash.update(filename) + return hash.hexdigest() + + def get_source_checksum(self, source): + """Returns a checksum for the source.""" + return sha1(source.encode("utf-8")).hexdigest() + + def get_bucket(self, environment, name, filename, source): + """Return a cache bucket for the given template. All arguments are + mandatory but filename may be `None`. + """ + key = self.get_cache_key(name, filename) + checksum = self.get_source_checksum(source) + bucket = Bucket(environment, key, checksum) + self.load_bytecode(bucket) + return bucket + + def set_bucket(self, bucket): + """Put the bucket into the cache.""" + self.dump_bytecode(bucket) + + +class FileSystemBytecodeCache(BytecodeCache): + """A bytecode cache that stores bytecode on the filesystem. It accepts + two arguments: The directory where the cache items are stored and a + pattern string that is used to build the filename. + + If no directory is specified a default cache directory is selected. On + Windows the user's temp directory is used, on UNIX systems a directory + is created for the user in the system temp directory. + + The pattern can be used to have multiple separate caches operate on the + same directory. The default pattern is ``'__jinja2_%s.cache'``. ``%s`` + is replaced with the cache key. + + >>> bcc = FileSystemBytecodeCache('/tmp/jinja_cache', '%s.cache') + + This bytecode cache supports clearing of the cache using the clear method. + """ + + def __init__(self, directory=None, pattern="__jinja2_%s.cache"): + if directory is None: + directory = self._get_default_cache_dir() + self.directory = directory + self.pattern = pattern + + def _get_default_cache_dir(self): + def _unsafe_dir(): + raise RuntimeError( + "Cannot determine safe temp directory. You " + "need to explicitly provide one." + ) + + tmpdir = tempfile.gettempdir() + + # On windows the temporary directory is used specific unless + # explicitly forced otherwise. We can just use that. + if os.name == "nt": + return tmpdir + if not hasattr(os, "getuid"): + _unsafe_dir() + + dirname = "_jinja2-cache-%d" % os.getuid() + actual_dir = os.path.join(tmpdir, dirname) + + try: + os.mkdir(actual_dir, stat.S_IRWXU) + except OSError as e: + if e.errno != errno.EEXIST: + raise + try: + os.chmod(actual_dir, stat.S_IRWXU) + actual_dir_stat = os.lstat(actual_dir) + if ( + actual_dir_stat.st_uid != os.getuid() + or not stat.S_ISDIR(actual_dir_stat.st_mode) + or stat.S_IMODE(actual_dir_stat.st_mode) != stat.S_IRWXU + ): + _unsafe_dir() + except OSError as e: + if e.errno != errno.EEXIST: + raise + + actual_dir_stat = os.lstat(actual_dir) + if ( + actual_dir_stat.st_uid != os.getuid() + or not stat.S_ISDIR(actual_dir_stat.st_mode) + or stat.S_IMODE(actual_dir_stat.st_mode) != stat.S_IRWXU + ): + _unsafe_dir() + + return actual_dir + + def _get_cache_filename(self, bucket): + return path.join(self.directory, self.pattern % bucket.key) + + def load_bytecode(self, bucket): + f = open_if_exists(self._get_cache_filename(bucket), "rb") + if f is not None: + try: + bucket.load_bytecode(f) + finally: + f.close() + + def dump_bytecode(self, bucket): + f = open(self._get_cache_filename(bucket), "wb") + try: + bucket.write_bytecode(f) + finally: + f.close() + + def clear(self): + # imported lazily here because google app-engine doesn't support + # write access on the file system and the function does not exist + # normally. + from os import remove + + files = fnmatch.filter(listdir(self.directory), self.pattern % "*") + for filename in files: + try: + remove(path.join(self.directory, filename)) + except OSError: + pass + + +class MemcachedBytecodeCache(BytecodeCache): + """This class implements a bytecode cache that uses a memcache cache for + storing the information. It does not enforce a specific memcache library + (tummy's memcache or cmemcache) but will accept any class that provides + the minimal interface required. + + Libraries compatible with this class: + + - `cachelib `_ + - `python-memcached `_ + + (Unfortunately the django cache interface is not compatible because it + does not support storing binary data, only unicode. You can however pass + the underlying cache client to the bytecode cache which is available + as `django.core.cache.cache._client`.) + + The minimal interface for the client passed to the constructor is this: + + .. class:: MinimalClientInterface + + .. method:: set(key, value[, timeout]) + + Stores the bytecode in the cache. `value` is a string and + `timeout` the timeout of the key. If timeout is not provided + a default timeout or no timeout should be assumed, if it's + provided it's an integer with the number of seconds the cache + item should exist. + + .. method:: get(key) + + Returns the value for the cache key. If the item does not + exist in the cache the return value must be `None`. + + The other arguments to the constructor are the prefix for all keys that + is added before the actual cache key and the timeout for the bytecode in + the cache system. We recommend a high (or no) timeout. + + This bytecode cache does not support clearing of used items in the cache. + The clear method is a no-operation function. + + .. versionadded:: 2.7 + Added support for ignoring memcache errors through the + `ignore_memcache_errors` parameter. + """ + + def __init__( + self, + client, + prefix="jinja2/bytecode/", + timeout=None, + ignore_memcache_errors=True, + ): + self.client = client + self.prefix = prefix + self.timeout = timeout + self.ignore_memcache_errors = ignore_memcache_errors + + def load_bytecode(self, bucket): + try: + code = self.client.get(self.prefix + bucket.key) + except Exception: + if not self.ignore_memcache_errors: + raise + code = None + if code is not None: + bucket.bytecode_from_string(code) + + def dump_bytecode(self, bucket): + args = (self.prefix + bucket.key, bucket.bytecode_to_string()) + if self.timeout is not None: + args += (self.timeout,) + try: + self.client.set(*args) + except Exception: + if not self.ignore_memcache_errors: + raise diff --git a/magma/squid_cnf/charms/squid/venv/jinja2/compiler.py b/magma/squid_cnf/charms/squid/venv/jinja2/compiler.py new file mode 100644 index 0000000000000000000000000000000000000000..63297b42c30f17b0c0ae08547047d070e7a53d3c --- /dev/null +++ b/magma/squid_cnf/charms/squid/venv/jinja2/compiler.py @@ -0,0 +1,1843 @@ +# -*- coding: utf-8 -*- +"""Compiles nodes from the parser into Python code.""" +from collections import namedtuple +from functools import update_wrapper +from itertools import chain +from keyword import iskeyword as is_python_keyword + +from markupsafe import escape +from markupsafe import Markup + +from . import nodes +from ._compat import imap +from ._compat import iteritems +from ._compat import izip +from ._compat import NativeStringIO +from ._compat import range_type +from ._compat import string_types +from ._compat import text_type +from .exceptions import TemplateAssertionError +from .idtracking import Symbols +from .idtracking import VAR_LOAD_ALIAS +from .idtracking import VAR_LOAD_PARAMETER +from .idtracking import VAR_LOAD_RESOLVE +from .idtracking import VAR_LOAD_UNDEFINED +from .nodes import EvalContext +from .optimizer import Optimizer +from .utils import concat +from .visitor import NodeVisitor + +operators = { + "eq": "==", + "ne": "!=", + "gt": ">", + "gteq": ">=", + "lt": "<", + "lteq": "<=", + "in": "in", + "notin": "not in", +} + +# what method to iterate over items do we want to use for dict iteration +# in generated code? on 2.x let's go with iteritems, on 3.x with items +if hasattr(dict, "iteritems"): + dict_item_iter = "iteritems" +else: + dict_item_iter = "items" + +code_features = ["division"] + +# does this python version support generator stops? (PEP 0479) +try: + exec("from __future__ import generator_stop") + code_features.append("generator_stop") +except SyntaxError: + pass + +# does this python version support yield from? +try: + exec("def f(): yield from x()") +except SyntaxError: + supports_yield_from = False +else: + supports_yield_from = True + + +def optimizeconst(f): + def new_func(self, node, frame, **kwargs): + # Only optimize if the frame is not volatile + if self.optimized and not frame.eval_ctx.volatile: + new_node = self.optimizer.visit(node, frame.eval_ctx) + if new_node != node: + return self.visit(new_node, frame) + return f(self, node, frame, **kwargs) + + return update_wrapper(new_func, f) + + +def generate( + node, environment, name, filename, stream=None, defer_init=False, optimized=True +): + """Generate the python source for a node tree.""" + if not isinstance(node, nodes.Template): + raise TypeError("Can't compile non template nodes") + generator = environment.code_generator_class( + environment, name, filename, stream, defer_init, optimized + ) + generator.visit(node) + if stream is None: + return generator.stream.getvalue() + + +def has_safe_repr(value): + """Does the node have a safe representation?""" + if value is None or value is NotImplemented or value is Ellipsis: + return True + if type(value) in (bool, int, float, complex, range_type, Markup) + string_types: + return True + if type(value) in (tuple, list, set, frozenset): + for item in value: + if not has_safe_repr(item): + return False + return True + elif type(value) is dict: + for key, value in iteritems(value): + if not has_safe_repr(key): + return False + if not has_safe_repr(value): + return False + return True + return False + + +def find_undeclared(nodes, names): + """Check if the names passed are accessed undeclared. The return value + is a set of all the undeclared names from the sequence of names found. + """ + visitor = UndeclaredNameVisitor(names) + try: + for node in nodes: + visitor.visit(node) + except VisitorExit: + pass + return visitor.undeclared + + +class MacroRef(object): + def __init__(self, node): + self.node = node + self.accesses_caller = False + self.accesses_kwargs = False + self.accesses_varargs = False + + +class Frame(object): + """Holds compile time information for us.""" + + def __init__(self, eval_ctx, parent=None, level=None): + self.eval_ctx = eval_ctx + self.symbols = Symbols(parent and parent.symbols or None, level=level) + + # a toplevel frame is the root + soft frames such as if conditions. + self.toplevel = False + + # the root frame is basically just the outermost frame, so no if + # conditions. This information is used to optimize inheritance + # situations. + self.rootlevel = False + + # in some dynamic inheritance situations the compiler needs to add + # write tests around output statements. + self.require_output_check = parent and parent.require_output_check + + # inside some tags we are using a buffer rather than yield statements. + # this for example affects {% filter %} or {% macro %}. If a frame + # is buffered this variable points to the name of the list used as + # buffer. + self.buffer = None + + # the name of the block we're in, otherwise None. + self.block = parent and parent.block or None + + # the parent of this frame + self.parent = parent + + if parent is not None: + self.buffer = parent.buffer + + def copy(self): + """Create a copy of the current one.""" + rv = object.__new__(self.__class__) + rv.__dict__.update(self.__dict__) + rv.symbols = self.symbols.copy() + return rv + + def inner(self, isolated=False): + """Return an inner frame.""" + if isolated: + return Frame(self.eval_ctx, level=self.symbols.level + 1) + return Frame(self.eval_ctx, self) + + def soft(self): + """Return a soft frame. A soft frame may not be modified as + standalone thing as it shares the resources with the frame it + was created of, but it's not a rootlevel frame any longer. + + This is only used to implement if-statements. + """ + rv = self.copy() + rv.rootlevel = False + return rv + + __copy__ = copy + + +class VisitorExit(RuntimeError): + """Exception used by the `UndeclaredNameVisitor` to signal a stop.""" + + +class DependencyFinderVisitor(NodeVisitor): + """A visitor that collects filter and test calls.""" + + def __init__(self): + self.filters = set() + self.tests = set() + + def visit_Filter(self, node): + self.generic_visit(node) + self.filters.add(node.name) + + def visit_Test(self, node): + self.generic_visit(node) + self.tests.add(node.name) + + def visit_Block(self, node): + """Stop visiting at blocks.""" + + +class UndeclaredNameVisitor(NodeVisitor): + """A visitor that checks if a name is accessed without being + declared. This is different from the frame visitor as it will + not stop at closure frames. + """ + + def __init__(self, names): + self.names = set(names) + self.undeclared = set() + + def visit_Name(self, node): + if node.ctx == "load" and node.name in self.names: + self.undeclared.add(node.name) + if self.undeclared == self.names: + raise VisitorExit() + else: + self.names.discard(node.name) + + def visit_Block(self, node): + """Stop visiting a blocks.""" + + +class CompilerExit(Exception): + """Raised if the compiler encountered a situation where it just + doesn't make sense to further process the code. Any block that + raises such an exception is not further processed. + """ + + +class CodeGenerator(NodeVisitor): + def __init__( + self, environment, name, filename, stream=None, defer_init=False, optimized=True + ): + if stream is None: + stream = NativeStringIO() + self.environment = environment + self.name = name + self.filename = filename + self.stream = stream + self.created_block_context = False + self.defer_init = defer_init + self.optimized = optimized + if optimized: + self.optimizer = Optimizer(environment) + + # aliases for imports + self.import_aliases = {} + + # a registry for all blocks. Because blocks are moved out + # into the global python scope they are registered here + self.blocks = {} + + # the number of extends statements so far + self.extends_so_far = 0 + + # some templates have a rootlevel extends. In this case we + # can safely assume that we're a child template and do some + # more optimizations. + self.has_known_extends = False + + # the current line number + self.code_lineno = 1 + + # registry of all filters and tests (global, not block local) + self.tests = {} + self.filters = {} + + # the debug information + self.debug_info = [] + self._write_debug_info = None + + # the number of new lines before the next write() + self._new_lines = 0 + + # the line number of the last written statement + self._last_line = 0 + + # true if nothing was written so far. + self._first_write = True + + # used by the `temporary_identifier` method to get new + # unique, temporary identifier + self._last_identifier = 0 + + # the current indentation + self._indentation = 0 + + # Tracks toplevel assignments + self._assign_stack = [] + + # Tracks parameter definition blocks + self._param_def_block = [] + + # Tracks the current context. + self._context_reference_stack = ["context"] + + # -- Various compilation helpers + + def fail(self, msg, lineno): + """Fail with a :exc:`TemplateAssertionError`.""" + raise TemplateAssertionError(msg, lineno, self.name, self.filename) + + def temporary_identifier(self): + """Get a new unique identifier.""" + self._last_identifier += 1 + return "t_%d" % self._last_identifier + + def buffer(self, frame): + """Enable buffering for the frame from that point onwards.""" + frame.buffer = self.temporary_identifier() + self.writeline("%s = []" % frame.buffer) + + def return_buffer_contents(self, frame, force_unescaped=False): + """Return the buffer contents of the frame.""" + if not force_unescaped: + if frame.eval_ctx.volatile: + self.writeline("if context.eval_ctx.autoescape:") + self.indent() + self.writeline("return Markup(concat(%s))" % frame.buffer) + self.outdent() + self.writeline("else:") + self.indent() + self.writeline("return concat(%s)" % frame.buffer) + self.outdent() + return + elif frame.eval_ctx.autoescape: + self.writeline("return Markup(concat(%s))" % frame.buffer) + return + self.writeline("return concat(%s)" % frame.buffer) + + def indent(self): + """Indent by one.""" + self._indentation += 1 + + def outdent(self, step=1): + """Outdent by step.""" + self._indentation -= step + + def start_write(self, frame, node=None): + """Yield or write into the frame buffer.""" + if frame.buffer is None: + self.writeline("yield ", node) + else: + self.writeline("%s.append(" % frame.buffer, node) + + def end_write(self, frame): + """End the writing process started by `start_write`.""" + if frame.buffer is not None: + self.write(")") + + def simple_write(self, s, frame, node=None): + """Simple shortcut for start_write + write + end_write.""" + self.start_write(frame, node) + self.write(s) + self.end_write(frame) + + def blockvisit(self, nodes, frame): + """Visit a list of nodes as block in a frame. If the current frame + is no buffer a dummy ``if 0: yield None`` is written automatically. + """ + try: + self.writeline("pass") + for node in nodes: + self.visit(node, frame) + except CompilerExit: + pass + + def write(self, x): + """Write a string into the output stream.""" + if self._new_lines: + if not self._first_write: + self.stream.write("\n" * self._new_lines) + self.code_lineno += self._new_lines + if self._write_debug_info is not None: + self.debug_info.append((self._write_debug_info, self.code_lineno)) + self._write_debug_info = None + self._first_write = False + self.stream.write(" " * self._indentation) + self._new_lines = 0 + self.stream.write(x) + + def writeline(self, x, node=None, extra=0): + """Combination of newline and write.""" + self.newline(node, extra) + self.write(x) + + def newline(self, node=None, extra=0): + """Add one or more newlines before the next write.""" + self._new_lines = max(self._new_lines, 1 + extra) + if node is not None and node.lineno != self._last_line: + self._write_debug_info = node.lineno + self._last_line = node.lineno + + def signature(self, node, frame, extra_kwargs=None): + """Writes a function call to the stream for the current node. + A leading comma is added automatically. The extra keyword + arguments may not include python keywords otherwise a syntax + error could occur. The extra keyword arguments should be given + as python dict. + """ + # if any of the given keyword arguments is a python keyword + # we have to make sure that no invalid call is created. + kwarg_workaround = False + for kwarg in chain((x.key for x in node.kwargs), extra_kwargs or ()): + if is_python_keyword(kwarg): + kwarg_workaround = True + break + + for arg in node.args: + self.write(", ") + self.visit(arg, frame) + + if not kwarg_workaround: + for kwarg in node.kwargs: + self.write(", ") + self.visit(kwarg, frame) + if extra_kwargs is not None: + for key, value in iteritems(extra_kwargs): + self.write(", %s=%s" % (key, value)) + if node.dyn_args: + self.write(", *") + self.visit(node.dyn_args, frame) + + if kwarg_workaround: + if node.dyn_kwargs is not None: + self.write(", **dict({") + else: + self.write(", **{") + for kwarg in node.kwargs: + self.write("%r: " % kwarg.key) + self.visit(kwarg.value, frame) + self.write(", ") + if extra_kwargs is not None: + for key, value in iteritems(extra_kwargs): + self.write("%r: %s, " % (key, value)) + if node.dyn_kwargs is not None: + self.write("}, **") + self.visit(node.dyn_kwargs, frame) + self.write(")") + else: + self.write("}") + + elif node.dyn_kwargs is not None: + self.write(", **") + self.visit(node.dyn_kwargs, frame) + + def pull_dependencies(self, nodes): + """Pull all the dependencies.""" + visitor = DependencyFinderVisitor() + for node in nodes: + visitor.visit(node) + for dependency in "filters", "tests": + mapping = getattr(self, dependency) + for name in getattr(visitor, dependency): + if name not in mapping: + mapping[name] = self.temporary_identifier() + self.writeline( + "%s = environment.%s[%r]" % (mapping[name], dependency, name) + ) + + def enter_frame(self, frame): + undefs = [] + for target, (action, param) in iteritems(frame.symbols.loads): + if action == VAR_LOAD_PARAMETER: + pass + elif action == VAR_LOAD_RESOLVE: + self.writeline("%s = %s(%r)" % (target, self.get_resolve_func(), param)) + elif action == VAR_LOAD_ALIAS: + self.writeline("%s = %s" % (target, param)) + elif action == VAR_LOAD_UNDEFINED: + undefs.append(target) + else: + raise NotImplementedError("unknown load instruction") + if undefs: + self.writeline("%s = missing" % " = ".join(undefs)) + + def leave_frame(self, frame, with_python_scope=False): + if not with_python_scope: + undefs = [] + for target, _ in iteritems(frame.symbols.loads): + undefs.append(target) + if undefs: + self.writeline("%s = missing" % " = ".join(undefs)) + + def func(self, name): + if self.environment.is_async: + return "async def %s" % name + return "def %s" % name + + def macro_body(self, node, frame): + """Dump the function def of a macro or call block.""" + frame = frame.inner() + frame.symbols.analyze_node(node) + macro_ref = MacroRef(node) + + explicit_caller = None + skip_special_params = set() + args = [] + for idx, arg in enumerate(node.args): + if arg.name == "caller": + explicit_caller = idx + if arg.name in ("kwargs", "varargs"): + skip_special_params.add(arg.name) + args.append(frame.symbols.ref(arg.name)) + + undeclared = find_undeclared(node.body, ("caller", "kwargs", "varargs")) + + if "caller" in undeclared: + # In older Jinja versions there was a bug that allowed caller + # to retain the special behavior even if it was mentioned in + # the argument list. However thankfully this was only really + # working if it was the last argument. So we are explicitly + # checking this now and error out if it is anywhere else in + # the argument list. + if explicit_caller is not None: + try: + node.defaults[explicit_caller - len(node.args)] + except IndexError: + self.fail( + "When defining macros or call blocks the " + 'special "caller" argument must be omitted ' + "or be given a default.", + node.lineno, + ) + else: + args.append(frame.symbols.declare_parameter("caller")) + macro_ref.accesses_caller = True + if "kwargs" in undeclared and "kwargs" not in skip_special_params: + args.append(frame.symbols.declare_parameter("kwargs")) + macro_ref.accesses_kwargs = True + if "varargs" in undeclared and "varargs" not in skip_special_params: + args.append(frame.symbols.declare_parameter("varargs")) + macro_ref.accesses_varargs = True + + # macros are delayed, they never require output checks + frame.require_output_check = False + frame.symbols.analyze_node(node) + self.writeline("%s(%s):" % (self.func("macro"), ", ".join(args)), node) + self.indent() + + self.buffer(frame) + self.enter_frame(frame) + + self.push_parameter_definitions(frame) + for idx, arg in enumerate(node.args): + ref = frame.symbols.ref(arg.name) + self.writeline("if %s is missing:" % ref) + self.indent() + try: + default = node.defaults[idx - len(node.args)] + except IndexError: + self.writeline( + "%s = undefined(%r, name=%r)" + % (ref, "parameter %r was not provided" % arg.name, arg.name) + ) + else: + self.writeline("%s = " % ref) + self.visit(default, frame) + self.mark_parameter_stored(ref) + self.outdent() + self.pop_parameter_definitions() + + self.blockvisit(node.body, frame) + self.return_buffer_contents(frame, force_unescaped=True) + self.leave_frame(frame, with_python_scope=True) + self.outdent() + + return frame, macro_ref + + def macro_def(self, macro_ref, frame): + """Dump the macro definition for the def created by macro_body.""" + arg_tuple = ", ".join(repr(x.name) for x in macro_ref.node.args) + name = getattr(macro_ref.node, "name", None) + if len(macro_ref.node.args) == 1: + arg_tuple += "," + self.write( + "Macro(environment, macro, %r, (%s), %r, %r, %r, " + "context.eval_ctx.autoescape)" + % ( + name, + arg_tuple, + macro_ref.accesses_kwargs, + macro_ref.accesses_varargs, + macro_ref.accesses_caller, + ) + ) + + def position(self, node): + """Return a human readable position for the node.""" + rv = "line %d" % node.lineno + if self.name is not None: + rv += " in " + repr(self.name) + return rv + + def dump_local_context(self, frame): + return "{%s}" % ", ".join( + "%r: %s" % (name, target) + for name, target in iteritems(frame.symbols.dump_stores()) + ) + + def write_commons(self): + """Writes a common preamble that is used by root and block functions. + Primarily this sets up common local helpers and enforces a generator + through a dead branch. + """ + self.writeline("resolve = context.resolve_or_missing") + self.writeline("undefined = environment.undefined") + # always use the standard Undefined class for the implicit else of + # conditional expressions + self.writeline("cond_expr_undefined = Undefined") + self.writeline("if 0: yield None") + + def push_parameter_definitions(self, frame): + """Pushes all parameter targets from the given frame into a local + stack that permits tracking of yet to be assigned parameters. In + particular this enables the optimization from `visit_Name` to skip + undefined expressions for parameters in macros as macros can reference + otherwise unbound parameters. + """ + self._param_def_block.append(frame.symbols.dump_param_targets()) + + def pop_parameter_definitions(self): + """Pops the current parameter definitions set.""" + self._param_def_block.pop() + + def mark_parameter_stored(self, target): + """Marks a parameter in the current parameter definitions as stored. + This will skip the enforced undefined checks. + """ + if self._param_def_block: + self._param_def_block[-1].discard(target) + + def push_context_reference(self, target): + self._context_reference_stack.append(target) + + def pop_context_reference(self): + self._context_reference_stack.pop() + + def get_context_ref(self): + return self._context_reference_stack[-1] + + def get_resolve_func(self): + target = self._context_reference_stack[-1] + if target == "context": + return "resolve" + return "%s.resolve" % target + + def derive_context(self, frame): + return "%s.derived(%s)" % ( + self.get_context_ref(), + self.dump_local_context(frame), + ) + + def parameter_is_undeclared(self, target): + """Checks if a given target is an undeclared parameter.""" + if not self._param_def_block: + return False + return target in self._param_def_block[-1] + + def push_assign_tracking(self): + """Pushes a new layer for assignment tracking.""" + self._assign_stack.append(set()) + + def pop_assign_tracking(self, frame): + """Pops the topmost level for assignment tracking and updates the + context variables if necessary. + """ + vars = self._assign_stack.pop() + if not frame.toplevel or not vars: + return + public_names = [x for x in vars if x[:1] != "_"] + if len(vars) == 1: + name = next(iter(vars)) + ref = frame.symbols.ref(name) + self.writeline("context.vars[%r] = %s" % (name, ref)) + else: + self.writeline("context.vars.update({") + for idx, name in enumerate(vars): + if idx: + self.write(", ") + ref = frame.symbols.ref(name) + self.write("%r: %s" % (name, ref)) + self.write("})") + if public_names: + if len(public_names) == 1: + self.writeline("context.exported_vars.add(%r)" % public_names[0]) + else: + self.writeline( + "context.exported_vars.update((%s))" + % ", ".join(imap(repr, public_names)) + ) + + # -- Statement Visitors + + def visit_Template(self, node, frame=None): + assert frame is None, "no root frame allowed" + eval_ctx = EvalContext(self.environment, self.name) + + from .runtime import exported + + self.writeline("from __future__ import %s" % ", ".join(code_features)) + self.writeline("from jinja2.runtime import " + ", ".join(exported)) + + if self.environment.is_async: + self.writeline( + "from jinja2.asyncsupport import auto_await, " + "auto_aiter, AsyncLoopContext" + ) + + # if we want a deferred initialization we cannot move the + # environment into a local name + envenv = not self.defer_init and ", environment=environment" or "" + + # do we have an extends tag at all? If not, we can save some + # overhead by just not processing any inheritance code. + have_extends = node.find(nodes.Extends) is not None + + # find all blocks + for block in node.find_all(nodes.Block): + if block.name in self.blocks: + self.fail("block %r defined twice" % block.name, block.lineno) + self.blocks[block.name] = block + + # find all imports and import them + for import_ in node.find_all(nodes.ImportedName): + if import_.importname not in self.import_aliases: + imp = import_.importname + self.import_aliases[imp] = alias = self.temporary_identifier() + if "." in imp: + module, obj = imp.rsplit(".", 1) + self.writeline("from %s import %s as %s" % (module, obj, alias)) + else: + self.writeline("import %s as %s" % (imp, alias)) + + # add the load name + self.writeline("name = %r" % self.name) + + # generate the root render function. + self.writeline( + "%s(context, missing=missing%s):" % (self.func("root"), envenv), extra=1 + ) + self.indent() + self.write_commons() + + # process the root + frame = Frame(eval_ctx) + if "self" in find_undeclared(node.body, ("self",)): + ref = frame.symbols.declare_parameter("self") + self.writeline("%s = TemplateReference(context)" % ref) + frame.symbols.analyze_node(node) + frame.toplevel = frame.rootlevel = True + frame.require_output_check = have_extends and not self.has_known_extends + if have_extends: + self.writeline("parent_template = None") + self.enter_frame(frame) + self.pull_dependencies(node.body) + self.blockvisit(node.body, frame) + self.leave_frame(frame, with_python_scope=True) + self.outdent() + + # make sure that the parent root is called. + if have_extends: + if not self.has_known_extends: + self.indent() + self.writeline("if parent_template is not None:") + self.indent() + if supports_yield_from and not self.environment.is_async: + self.writeline("yield from parent_template.root_render_func(context)") + else: + self.writeline( + "%sfor event in parent_template." + "root_render_func(context):" + % (self.environment.is_async and "async " or "") + ) + self.indent() + self.writeline("yield event") + self.outdent() + self.outdent(1 + (not self.has_known_extends)) + + # at this point we now have the blocks collected and can visit them too. + for name, block in iteritems(self.blocks): + self.writeline( + "%s(context, missing=missing%s):" + % (self.func("block_" + name), envenv), + block, + 1, + ) + self.indent() + self.write_commons() + # It's important that we do not make this frame a child of the + # toplevel template. This would cause a variety of + # interesting issues with identifier tracking. + block_frame = Frame(eval_ctx) + undeclared = find_undeclared(block.body, ("self", "super")) + if "self" in undeclared: + ref = block_frame.symbols.declare_parameter("self") + self.writeline("%s = TemplateReference(context)" % ref) + if "super" in undeclared: + ref = block_frame.symbols.declare_parameter("super") + self.writeline("%s = context.super(%r, block_%s)" % (ref, name, name)) + block_frame.symbols.analyze_node(block) + block_frame.block = name + self.enter_frame(block_frame) + self.pull_dependencies(block.body) + self.blockvisit(block.body, block_frame) + self.leave_frame(block_frame, with_python_scope=True) + self.outdent() + + self.writeline( + "blocks = {%s}" % ", ".join("%r: block_%s" % (x, x) for x in self.blocks), + extra=1, + ) + + # add a function that returns the debug info + self.writeline( + "debug_info = %r" % "&".join("%s=%s" % x for x in self.debug_info) + ) + + def visit_Block(self, node, frame): + """Call a block and register it for the template.""" + level = 0 + if frame.toplevel: + # if we know that we are a child template, there is no need to + # check if we are one + if self.has_known_extends: + return + if self.extends_so_far > 0: + self.writeline("if parent_template is None:") + self.indent() + level += 1 + + if node.scoped: + context = self.derive_context(frame) + else: + context = self.get_context_ref() + + if ( + supports_yield_from + and not self.environment.is_async + and frame.buffer is None + ): + self.writeline( + "yield from context.blocks[%r][0](%s)" % (node.name, context), node + ) + else: + loop = self.environment.is_async and "async for" or "for" + self.writeline( + "%s event in context.blocks[%r][0](%s):" % (loop, node.name, context), + node, + ) + self.indent() + self.simple_write("event", frame) + self.outdent() + + self.outdent(level) + + def visit_Extends(self, node, frame): + """Calls the extender.""" + if not frame.toplevel: + self.fail("cannot use extend from a non top-level scope", node.lineno) + + # if the number of extends statements in general is zero so + # far, we don't have to add a check if something extended + # the template before this one. + if self.extends_so_far > 0: + + # if we have a known extends we just add a template runtime + # error into the generated code. We could catch that at compile + # time too, but i welcome it not to confuse users by throwing the + # same error at different times just "because we can". + if not self.has_known_extends: + self.writeline("if parent_template is not None:") + self.indent() + self.writeline("raise TemplateRuntimeError(%r)" % "extended multiple times") + + # if we have a known extends already we don't need that code here + # as we know that the template execution will end here. + if self.has_known_extends: + raise CompilerExit() + else: + self.outdent() + + self.writeline("parent_template = environment.get_template(", node) + self.visit(node.template, frame) + self.write(", %r)" % self.name) + self.writeline( + "for name, parent_block in parent_template.blocks.%s():" % dict_item_iter + ) + self.indent() + self.writeline("context.blocks.setdefault(name, []).append(parent_block)") + self.outdent() + + # if this extends statement was in the root level we can take + # advantage of that information and simplify the generated code + # in the top level from this point onwards + if frame.rootlevel: + self.has_known_extends = True + + # and now we have one more + self.extends_so_far += 1 + + def visit_Include(self, node, frame): + """Handles includes.""" + if node.ignore_missing: + self.writeline("try:") + self.indent() + + func_name = "get_or_select_template" + if isinstance(node.template, nodes.Const): + if isinstance(node.template.value, string_types): + func_name = "get_template" + elif isinstance(node.template.value, (tuple, list)): + func_name = "select_template" + elif isinstance(node.template, (nodes.Tuple, nodes.List)): + func_name = "select_template" + + self.writeline("template = environment.%s(" % func_name, node) + self.visit(node.template, frame) + self.write(", %r)" % self.name) + if node.ignore_missing: + self.outdent() + self.writeline("except TemplateNotFound:") + self.indent() + self.writeline("pass") + self.outdent() + self.writeline("else:") + self.indent() + + skip_event_yield = False + if node.with_context: + loop = self.environment.is_async and "async for" or "for" + self.writeline( + "%s event in template.root_render_func(" + "template.new_context(context.get_all(), True, " + "%s)):" % (loop, self.dump_local_context(frame)) + ) + elif self.environment.is_async: + self.writeline( + "for event in (await " + "template._get_default_module_async())" + "._body_stream:" + ) + else: + if supports_yield_from: + self.writeline("yield from template._get_default_module()._body_stream") + skip_event_yield = True + else: + self.writeline( + "for event in template._get_default_module()._body_stream:" + ) + + if not skip_event_yield: + self.indent() + self.simple_write("event", frame) + self.outdent() + + if node.ignore_missing: + self.outdent() + + def visit_Import(self, node, frame): + """Visit regular imports.""" + self.writeline("%s = " % frame.symbols.ref(node.target), node) + if frame.toplevel: + self.write("context.vars[%r] = " % node.target) + if self.environment.is_async: + self.write("await ") + self.write("environment.get_template(") + self.visit(node.template, frame) + self.write(", %r)." % self.name) + if node.with_context: + self.write( + "make_module%s(context.get_all(), True, %s)" + % ( + self.environment.is_async and "_async" or "", + self.dump_local_context(frame), + ) + ) + elif self.environment.is_async: + self.write("_get_default_module_async()") + else: + self.write("_get_default_module()") + if frame.toplevel and not node.target.startswith("_"): + self.writeline("context.exported_vars.discard(%r)" % node.target) + + def visit_FromImport(self, node, frame): + """Visit named imports.""" + self.newline(node) + self.write( + "included_template = %senvironment.get_template(" + % (self.environment.is_async and "await " or "") + ) + self.visit(node.template, frame) + self.write(", %r)." % self.name) + if node.with_context: + self.write( + "make_module%s(context.get_all(), True, %s)" + % ( + self.environment.is_async and "_async" or "", + self.dump_local_context(frame), + ) + ) + elif self.environment.is_async: + self.write("_get_default_module_async()") + else: + self.write("_get_default_module()") + + var_names = [] + discarded_names = [] + for name in node.names: + if isinstance(name, tuple): + name, alias = name + else: + alias = name + self.writeline( + "%s = getattr(included_template, " + "%r, missing)" % (frame.symbols.ref(alias), name) + ) + self.writeline("if %s is missing:" % frame.symbols.ref(alias)) + self.indent() + self.writeline( + "%s = undefined(%r %% " + "included_template.__name__, " + "name=%r)" + % ( + frame.symbols.ref(alias), + "the template %%r (imported on %s) does " + "not export the requested name %s" + % (self.position(node), repr(name)), + name, + ) + ) + self.outdent() + if frame.toplevel: + var_names.append(alias) + if not alias.startswith("_"): + discarded_names.append(alias) + + if var_names: + if len(var_names) == 1: + name = var_names[0] + self.writeline( + "context.vars[%r] = %s" % (name, frame.symbols.ref(name)) + ) + else: + self.writeline( + "context.vars.update({%s})" + % ", ".join( + "%r: %s" % (name, frame.symbols.ref(name)) for name in var_names + ) + ) + if discarded_names: + if len(discarded_names) == 1: + self.writeline("context.exported_vars.discard(%r)" % discarded_names[0]) + else: + self.writeline( + "context.exported_vars.difference_" + "update((%s))" % ", ".join(imap(repr, discarded_names)) + ) + + def visit_For(self, node, frame): + loop_frame = frame.inner() + test_frame = frame.inner() + else_frame = frame.inner() + + # try to figure out if we have an extended loop. An extended loop + # is necessary if the loop is in recursive mode if the special loop + # variable is accessed in the body. + extended_loop = node.recursive or "loop" in find_undeclared( + node.iter_child_nodes(only=("body",)), ("loop",) + ) + + loop_ref = None + if extended_loop: + loop_ref = loop_frame.symbols.declare_parameter("loop") + + loop_frame.symbols.analyze_node(node, for_branch="body") + if node.else_: + else_frame.symbols.analyze_node(node, for_branch="else") + + if node.test: + loop_filter_func = self.temporary_identifier() + test_frame.symbols.analyze_node(node, for_branch="test") + self.writeline("%s(fiter):" % self.func(loop_filter_func), node.test) + self.indent() + self.enter_frame(test_frame) + self.writeline(self.environment.is_async and "async for " or "for ") + self.visit(node.target, loop_frame) + self.write(" in ") + self.write(self.environment.is_async and "auto_aiter(fiter)" or "fiter") + self.write(":") + self.indent() + self.writeline("if ", node.test) + self.visit(node.test, test_frame) + self.write(":") + self.indent() + self.writeline("yield ") + self.visit(node.target, loop_frame) + self.outdent(3) + self.leave_frame(test_frame, with_python_scope=True) + + # if we don't have an recursive loop we have to find the shadowed + # variables at that point. Because loops can be nested but the loop + # variable is a special one we have to enforce aliasing for it. + if node.recursive: + self.writeline( + "%s(reciter, loop_render_func, depth=0):" % self.func("loop"), node + ) + self.indent() + self.buffer(loop_frame) + + # Use the same buffer for the else frame + else_frame.buffer = loop_frame.buffer + + # make sure the loop variable is a special one and raise a template + # assertion error if a loop tries to write to loop + if extended_loop: + self.writeline("%s = missing" % loop_ref) + + for name in node.find_all(nodes.Name): + if name.ctx == "store" and name.name == "loop": + self.fail( + "Can't assign to special loop variable in for-loop target", + name.lineno, + ) + + if node.else_: + iteration_indicator = self.temporary_identifier() + self.writeline("%s = 1" % iteration_indicator) + + self.writeline(self.environment.is_async and "async for " or "for ", node) + self.visit(node.target, loop_frame) + if extended_loop: + if self.environment.is_async: + self.write(", %s in AsyncLoopContext(" % loop_ref) + else: + self.write(", %s in LoopContext(" % loop_ref) + else: + self.write(" in ") + + if node.test: + self.write("%s(" % loop_filter_func) + if node.recursive: + self.write("reciter") + else: + if self.environment.is_async and not extended_loop: + self.write("auto_aiter(") + self.visit(node.iter, frame) + if self.environment.is_async and not extended_loop: + self.write(")") + if node.test: + self.write(")") + + if node.recursive: + self.write(", undefined, loop_render_func, depth):") + else: + self.write(extended_loop and ", undefined):" or ":") + + self.indent() + self.enter_frame(loop_frame) + + self.blockvisit(node.body, loop_frame) + if node.else_: + self.writeline("%s = 0" % iteration_indicator) + self.outdent() + self.leave_frame( + loop_frame, with_python_scope=node.recursive and not node.else_ + ) + + if node.else_: + self.writeline("if %s:" % iteration_indicator) + self.indent() + self.enter_frame(else_frame) + self.blockvisit(node.else_, else_frame) + self.leave_frame(else_frame) + self.outdent() + + # if the node was recursive we have to return the buffer contents + # and start the iteration code + if node.recursive: + self.return_buffer_contents(loop_frame) + self.outdent() + self.start_write(frame, node) + if self.environment.is_async: + self.write("await ") + self.write("loop(") + if self.environment.is_async: + self.write("auto_aiter(") + self.visit(node.iter, frame) + if self.environment.is_async: + self.write(")") + self.write(", loop)") + self.end_write(frame) + + def visit_If(self, node, frame): + if_frame = frame.soft() + self.writeline("if ", node) + self.visit(node.test, if_frame) + self.write(":") + self.indent() + self.blockvisit(node.body, if_frame) + self.outdent() + for elif_ in node.elif_: + self.writeline("elif ", elif_) + self.visit(elif_.test, if_frame) + self.write(":") + self.indent() + self.blockvisit(elif_.body, if_frame) + self.outdent() + if node.else_: + self.writeline("else:") + self.indent() + self.blockvisit(node.else_, if_frame) + self.outdent() + + def visit_Macro(self, node, frame): + macro_frame, macro_ref = self.macro_body(node, frame) + self.newline() + if frame.toplevel: + if not node.name.startswith("_"): + self.write("context.exported_vars.add(%r)" % node.name) + self.writeline("context.vars[%r] = " % node.name) + self.write("%s = " % frame.symbols.ref(node.name)) + self.macro_def(macro_ref, macro_frame) + + def visit_CallBlock(self, node, frame): + call_frame, macro_ref = self.macro_body(node, frame) + self.writeline("caller = ") + self.macro_def(macro_ref, call_frame) + self.start_write(frame, node) + self.visit_Call(node.call, frame, forward_caller=True) + self.end_write(frame) + + def visit_FilterBlock(self, node, frame): + filter_frame = frame.inner() + filter_frame.symbols.analyze_node(node) + self.enter_frame(filter_frame) + self.buffer(filter_frame) + self.blockvisit(node.body, filter_frame) + self.start_write(frame, node) + self.visit_Filter(node.filter, filter_frame) + self.end_write(frame) + self.leave_frame(filter_frame) + + def visit_With(self, node, frame): + with_frame = frame.inner() + with_frame.symbols.analyze_node(node) + self.enter_frame(with_frame) + for target, expr in izip(node.targets, node.values): + self.newline() + self.visit(target, with_frame) + self.write(" = ") + self.visit(expr, frame) + self.blockvisit(node.body, with_frame) + self.leave_frame(with_frame) + + def visit_ExprStmt(self, node, frame): + self.newline(node) + self.visit(node.node, frame) + + _FinalizeInfo = namedtuple("_FinalizeInfo", ("const", "src")) + #: The default finalize function if the environment isn't configured + #: with one. Or if the environment has one, this is called on that + #: function's output for constants. + _default_finalize = text_type + _finalize = None + + def _make_finalize(self): + """Build the finalize function to be used on constants and at + runtime. Cached so it's only created once for all output nodes. + + Returns a ``namedtuple`` with the following attributes: + + ``const`` + A function to finalize constant data at compile time. + + ``src`` + Source code to output around nodes to be evaluated at + runtime. + """ + if self._finalize is not None: + return self._finalize + + finalize = default = self._default_finalize + src = None + + if self.environment.finalize: + src = "environment.finalize(" + env_finalize = self.environment.finalize + + def finalize(value): + return default(env_finalize(value)) + + if getattr(env_finalize, "contextfunction", False) is True: + src += "context, " + finalize = None # noqa: F811 + elif getattr(env_finalize, "evalcontextfunction", False) is True: + src += "context.eval_ctx, " + finalize = None + elif getattr(env_finalize, "environmentfunction", False) is True: + src += "environment, " + + def finalize(value): + return default(env_finalize(self.environment, value)) + + self._finalize = self._FinalizeInfo(finalize, src) + return self._finalize + + def _output_const_repr(self, group): + """Given a group of constant values converted from ``Output`` + child nodes, produce a string to write to the template module + source. + """ + return repr(concat(group)) + + def _output_child_to_const(self, node, frame, finalize): + """Try to optimize a child of an ``Output`` node by trying to + convert it to constant, finalized data at compile time. + + If :exc:`Impossible` is raised, the node is not constant and + will be evaluated at runtime. Any other exception will also be + evaluated at runtime for easier debugging. + """ + const = node.as_const(frame.eval_ctx) + + if frame.eval_ctx.autoescape: + const = escape(const) + + # Template data doesn't go through finalize. + if isinstance(node, nodes.TemplateData): + return text_type(const) + + return finalize.const(const) + + def _output_child_pre(self, node, frame, finalize): + """Output extra source code before visiting a child of an + ``Output`` node. + """ + if frame.eval_ctx.volatile: + self.write("(escape if context.eval_ctx.autoescape else to_string)(") + elif frame.eval_ctx.autoescape: + self.write("escape(") + else: + self.write("to_string(") + + if finalize.src is not None: + self.write(finalize.src) + + def _output_child_post(self, node, frame, finalize): + """Output extra source code after visiting a child of an + ``Output`` node. + """ + self.write(")") + + if finalize.src is not None: + self.write(")") + + def visit_Output(self, node, frame): + # If an extends is active, don't render outside a block. + if frame.require_output_check: + # A top-level extends is known to exist at compile time. + if self.has_known_extends: + return + + self.writeline("if parent_template is None:") + self.indent() + + finalize = self._make_finalize() + body = [] + + # Evaluate constants at compile time if possible. Each item in + # body will be either a list of static data or a node to be + # evaluated at runtime. + for child in node.nodes: + try: + if not ( + # If the finalize function requires runtime context, + # constants can't be evaluated at compile time. + finalize.const + # Unless it's basic template data that won't be + # finalized anyway. + or isinstance(child, nodes.TemplateData) + ): + raise nodes.Impossible() + + const = self._output_child_to_const(child, frame, finalize) + except (nodes.Impossible, Exception): + # The node was not constant and needs to be evaluated at + # runtime. Or another error was raised, which is easier + # to debug at runtime. + body.append(child) + continue + + if body and isinstance(body[-1], list): + body[-1].append(const) + else: + body.append([const]) + + if frame.buffer is not None: + if len(body) == 1: + self.writeline("%s.append(" % frame.buffer) + else: + self.writeline("%s.extend((" % frame.buffer) + + self.indent() + + for item in body: + if isinstance(item, list): + # A group of constant data to join and output. + val = self._output_const_repr(item) + + if frame.buffer is None: + self.writeline("yield " + val) + else: + self.writeline(val + ",") + else: + if frame.buffer is None: + self.writeline("yield ", item) + else: + self.newline(item) + + # A node to be evaluated at runtime. + self._output_child_pre(item, frame, finalize) + self.visit(item, frame) + self._output_child_post(item, frame, finalize) + + if frame.buffer is not None: + self.write(",") + + if frame.buffer is not None: + self.outdent() + self.writeline(")" if len(body) == 1 else "))") + + if frame.require_output_check: + self.outdent() + + def visit_Assign(self, node, frame): + self.push_assign_tracking() + self.newline(node) + self.visit(node.target, frame) + self.write(" = ") + self.visit(node.node, frame) + self.pop_assign_tracking(frame) + + def visit_AssignBlock(self, node, frame): + self.push_assign_tracking() + block_frame = frame.inner() + # This is a special case. Since a set block always captures we + # will disable output checks. This way one can use set blocks + # toplevel even in extended templates. + block_frame.require_output_check = False + block_frame.symbols.analyze_node(node) + self.enter_frame(block_frame) + self.buffer(block_frame) + self.blockvisit(node.body, block_frame) + self.newline(node) + self.visit(node.target, frame) + self.write(" = (Markup if context.eval_ctx.autoescape else identity)(") + if node.filter is not None: + self.visit_Filter(node.filter, block_frame) + else: + self.write("concat(%s)" % block_frame.buffer) + self.write(")") + self.pop_assign_tracking(frame) + self.leave_frame(block_frame) + + # -- Expression Visitors + + def visit_Name(self, node, frame): + if node.ctx == "store" and frame.toplevel: + if self._assign_stack: + self._assign_stack[-1].add(node.name) + ref = frame.symbols.ref(node.name) + + # If we are looking up a variable we might have to deal with the + # case where it's undefined. We can skip that case if the load + # instruction indicates a parameter which are always defined. + if node.ctx == "load": + load = frame.symbols.find_load(ref) + if not ( + load is not None + and load[0] == VAR_LOAD_PARAMETER + and not self.parameter_is_undeclared(ref) + ): + self.write( + "(undefined(name=%r) if %s is missing else %s)" + % (node.name, ref, ref) + ) + return + + self.write(ref) + + def visit_NSRef(self, node, frame): + # NSRefs can only be used to store values; since they use the normal + # `foo.bar` notation they will be parsed as a normal attribute access + # when used anywhere but in a `set` context + ref = frame.symbols.ref(node.name) + self.writeline("if not isinstance(%s, Namespace):" % ref) + self.indent() + self.writeline( + "raise TemplateRuntimeError(%r)" + % "cannot assign attribute on non-namespace object" + ) + self.outdent() + self.writeline("%s[%r]" % (ref, node.attr)) + + def visit_Const(self, node, frame): + val = node.as_const(frame.eval_ctx) + if isinstance(val, float): + self.write(str(val)) + else: + self.write(repr(val)) + + def visit_TemplateData(self, node, frame): + try: + self.write(repr(node.as_const(frame.eval_ctx))) + except nodes.Impossible: + self.write( + "(Markup if context.eval_ctx.autoescape else identity)(%r)" % node.data + ) + + def visit_Tuple(self, node, frame): + self.write("(") + idx = -1 + for idx, item in enumerate(node.items): + if idx: + self.write(", ") + self.visit(item, frame) + self.write(idx == 0 and ",)" or ")") + + def visit_List(self, node, frame): + self.write("[") + for idx, item in enumerate(node.items): + if idx: + self.write(", ") + self.visit(item, frame) + self.write("]") + + def visit_Dict(self, node, frame): + self.write("{") + for idx, item in enumerate(node.items): + if idx: + self.write(", ") + self.visit(item.key, frame) + self.write(": ") + self.visit(item.value, frame) + self.write("}") + + def binop(operator, interceptable=True): # noqa: B902 + @optimizeconst + def visitor(self, node, frame): + if ( + self.environment.sandboxed + and operator in self.environment.intercepted_binops + ): + self.write("environment.call_binop(context, %r, " % operator) + self.visit(node.left, frame) + self.write(", ") + self.visit(node.right, frame) + else: + self.write("(") + self.visit(node.left, frame) + self.write(" %s " % operator) + self.visit(node.right, frame) + self.write(")") + + return visitor + + def uaop(operator, interceptable=True): # noqa: B902 + @optimizeconst + def visitor(self, node, frame): + if ( + self.environment.sandboxed + and operator in self.environment.intercepted_unops + ): + self.write("environment.call_unop(context, %r, " % operator) + self.visit(node.node, frame) + else: + self.write("(" + operator) + self.visit(node.node, frame) + self.write(")") + + return visitor + + visit_Add = binop("+") + visit_Sub = binop("-") + visit_Mul = binop("*") + visit_Div = binop("/") + visit_FloorDiv = binop("//") + visit_Pow = binop("**") + visit_Mod = binop("%") + visit_And = binop("and", interceptable=False) + visit_Or = binop("or", interceptable=False) + visit_Pos = uaop("+") + visit_Neg = uaop("-") + visit_Not = uaop("not ", interceptable=False) + del binop, uaop + + @optimizeconst + def visit_Concat(self, node, frame): + if frame.eval_ctx.volatile: + func_name = "(context.eval_ctx.volatile and markup_join or unicode_join)" + elif frame.eval_ctx.autoescape: + func_name = "markup_join" + else: + func_name = "unicode_join" + self.write("%s((" % func_name) + for arg in node.nodes: + self.visit(arg, frame) + self.write(", ") + self.write("))") + + @optimizeconst + def visit_Compare(self, node, frame): + self.write("(") + self.visit(node.expr, frame) + for op in node.ops: + self.visit(op, frame) + self.write(")") + + def visit_Operand(self, node, frame): + self.write(" %s " % operators[node.op]) + self.visit(node.expr, frame) + + @optimizeconst + def visit_Getattr(self, node, frame): + if self.environment.is_async: + self.write("(await auto_await(") + + self.write("environment.getattr(") + self.visit(node.node, frame) + self.write(", %r)" % node.attr) + + if self.environment.is_async: + self.write("))") + + @optimizeconst + def visit_Getitem(self, node, frame): + # slices bypass the environment getitem method. + if isinstance(node.arg, nodes.Slice): + self.visit(node.node, frame) + self.write("[") + self.visit(node.arg, frame) + self.write("]") + else: + if self.environment.is_async: + self.write("(await auto_await(") + + self.write("environment.getitem(") + self.visit(node.node, frame) + self.write(", ") + self.visit(node.arg, frame) + self.write(")") + + if self.environment.is_async: + self.write("))") + + def visit_Slice(self, node, frame): + if node.start is not None: + self.visit(node.start, frame) + self.write(":") + if node.stop is not None: + self.visit(node.stop, frame) + if node.step is not None: + self.write(":") + self.visit(node.step, frame) + + @optimizeconst + def visit_Filter(self, node, frame): + if self.environment.is_async: + self.write("await auto_await(") + self.write(self.filters[node.name] + "(") + func = self.environment.filters.get(node.name) + if func is None: + self.fail("no filter named %r" % node.name, node.lineno) + if getattr(func, "contextfilter", False) is True: + self.write("context, ") + elif getattr(func, "evalcontextfilter", False) is True: + self.write("context.eval_ctx, ") + elif getattr(func, "environmentfilter", False) is True: + self.write("environment, ") + + # if the filter node is None we are inside a filter block + # and want to write to the current buffer + if node.node is not None: + self.visit(node.node, frame) + elif frame.eval_ctx.volatile: + self.write( + "(context.eval_ctx.autoescape and" + " Markup(concat(%s)) or concat(%s))" % (frame.buffer, frame.buffer) + ) + elif frame.eval_ctx.autoescape: + self.write("Markup(concat(%s))" % frame.buffer) + else: + self.write("concat(%s)" % frame.buffer) + self.signature(node, frame) + self.write(")") + if self.environment.is_async: + self.write(")") + + @optimizeconst + def visit_Test(self, node, frame): + self.write(self.tests[node.name] + "(") + if node.name not in self.environment.tests: + self.fail("no test named %r" % node.name, node.lineno) + self.visit(node.node, frame) + self.signature(node, frame) + self.write(")") + + @optimizeconst + def visit_CondExpr(self, node, frame): + def write_expr2(): + if node.expr2 is not None: + return self.visit(node.expr2, frame) + self.write( + "cond_expr_undefined(%r)" + % ( + "the inline if-" + "expression on %s evaluated to false and " + "no else section was defined." % self.position(node) + ) + ) + + self.write("(") + self.visit(node.expr1, frame) + self.write(" if ") + self.visit(node.test, frame) + self.write(" else ") + write_expr2() + self.write(")") + + @optimizeconst + def visit_Call(self, node, frame, forward_caller=False): + if self.environment.is_async: + self.write("await auto_await(") + if self.environment.sandboxed: + self.write("environment.call(context, ") + else: + self.write("context.call(") + self.visit(node.node, frame) + extra_kwargs = forward_caller and {"caller": "caller"} or None + self.signature(node, frame, extra_kwargs) + self.write(")") + if self.environment.is_async: + self.write(")") + + def visit_Keyword(self, node, frame): + self.write(node.key + "=") + self.visit(node.value, frame) + + # -- Unused nodes for extensions + + def visit_MarkSafe(self, node, frame): + self.write("Markup(") + self.visit(node.expr, frame) + self.write(")") + + def visit_MarkSafeIfAutoescape(self, node, frame): + self.write("(context.eval_ctx.autoescape and Markup or identity)(") + self.visit(node.expr, frame) + self.write(")") + + def visit_EnvironmentAttribute(self, node, frame): + self.write("environment." + node.name) + + def visit_ExtensionAttribute(self, node, frame): + self.write("environment.extensions[%r].%s" % (node.identifier, node.name)) + + def visit_ImportedName(self, node, frame): + self.write(self.import_aliases[node.importname]) + + def visit_InternalName(self, node, frame): + self.write(node.name) + + def visit_ContextReference(self, node, frame): + self.write("context") + + def visit_DerivedContextReference(self, node, frame): + self.write(self.derive_context(frame)) + + def visit_Continue(self, node, frame): + self.writeline("continue", node) + + def visit_Break(self, node, frame): + self.writeline("break", node) + + def visit_Scope(self, node, frame): + scope_frame = frame.inner() + scope_frame.symbols.analyze_node(node) + self.enter_frame(scope_frame) + self.blockvisit(node.body, scope_frame) + self.leave_frame(scope_frame) + + def visit_OverlayScope(self, node, frame): + ctx = self.temporary_identifier() + self.writeline("%s = %s" % (ctx, self.derive_context(frame))) + self.writeline("%s.vars = " % ctx) + self.visit(node.context, frame) + self.push_context_reference(ctx) + + scope_frame = frame.inner(isolated=True) + scope_frame.symbols.analyze_node(node) + self.enter_frame(scope_frame) + self.blockvisit(node.body, scope_frame) + self.leave_frame(scope_frame) + self.pop_context_reference() + + def visit_EvalContextModifier(self, node, frame): + for keyword in node.options: + self.writeline("context.eval_ctx.%s = " % keyword.key) + self.visit(keyword.value, frame) + try: + val = keyword.value.as_const(frame.eval_ctx) + except nodes.Impossible: + frame.eval_ctx.volatile = True + else: + setattr(frame.eval_ctx, keyword.key, val) + + def visit_ScopedEvalContextModifier(self, node, frame): + old_ctx_name = self.temporary_identifier() + saved_ctx = frame.eval_ctx.save() + self.writeline("%s = context.eval_ctx.save()" % old_ctx_name) + self.visit_EvalContextModifier(node, frame) + for child in node.body: + self.visit(child, frame) + frame.eval_ctx.revert(saved_ctx) + self.writeline("context.eval_ctx.revert(%s)" % old_ctx_name) diff --git a/magma/squid_cnf/charms/squid/venv/jinja2/constants.py b/magma/squid_cnf/charms/squid/venv/jinja2/constants.py new file mode 100644 index 0000000000000000000000000000000000000000..bf7f2ca721789052f1e227c5e3432e7712134c55 --- /dev/null +++ b/magma/squid_cnf/charms/squid/venv/jinja2/constants.py @@ -0,0 +1,21 @@ +# -*- coding: utf-8 -*- +#: list of lorem ipsum words used by the lipsum() helper function +LOREM_IPSUM_WORDS = u"""\ +a ac accumsan ad adipiscing aenean aliquam aliquet amet ante aptent arcu at +auctor augue bibendum blandit class commodo condimentum congue consectetuer +consequat conubia convallis cras cubilia cum curabitur curae cursus dapibus +diam dictum dictumst dignissim dis dolor donec dui duis egestas eget eleifend +elementum elit enim erat eros est et etiam eu euismod facilisi facilisis fames +faucibus felis fermentum feugiat fringilla fusce gravida habitant habitasse hac +hendrerit hymenaeos iaculis id imperdiet in inceptos integer interdum ipsum +justo lacinia lacus laoreet lectus leo libero ligula litora lobortis lorem +luctus maecenas magna magnis malesuada massa mattis mauris metus mi molestie +mollis montes morbi mus nam nascetur natoque nec neque netus nibh nisi nisl non +nonummy nostra nulla nullam nunc odio orci ornare parturient pede pellentesque +penatibus per pharetra phasellus placerat platea porta porttitor posuere +potenti praesent pretium primis proin pulvinar purus quam quis quisque rhoncus +ridiculus risus rutrum sagittis sapien scelerisque sed sem semper senectus sit +sociis sociosqu sodales sollicitudin suscipit suspendisse taciti tellus tempor +tempus tincidunt torquent tortor tristique turpis ullamcorper ultrices +ultricies urna ut varius vehicula vel velit venenatis vestibulum vitae vivamus +viverra volutpat vulputate""" diff --git a/magma/squid_cnf/charms/squid/venv/jinja2/debug.py b/magma/squid_cnf/charms/squid/venv/jinja2/debug.py new file mode 100644 index 0000000000000000000000000000000000000000..5d8aec31d05dab0b01b192b9dfe74f2656dc3077 --- /dev/null +++ b/magma/squid_cnf/charms/squid/venv/jinja2/debug.py @@ -0,0 +1,268 @@ +import sys +from types import CodeType + +from . import TemplateSyntaxError +from ._compat import PYPY +from .utils import internal_code +from .utils import missing + + +def rewrite_traceback_stack(source=None): + """Rewrite the current exception to replace any tracebacks from + within compiled template code with tracebacks that look like they + came from the template source. + + This must be called within an ``except`` block. + + :param exc_info: A :meth:`sys.exc_info` tuple. If not provided, + the current ``exc_info`` is used. + :param source: For ``TemplateSyntaxError``, the original source if + known. + :return: A :meth:`sys.exc_info` tuple that can be re-raised. + """ + exc_type, exc_value, tb = sys.exc_info() + + if isinstance(exc_value, TemplateSyntaxError) and not exc_value.translated: + exc_value.translated = True + exc_value.source = source + + try: + # Remove the old traceback on Python 3, otherwise the frames + # from the compiler still show up. + exc_value.with_traceback(None) + except AttributeError: + pass + + # Outside of runtime, so the frame isn't executing template + # code, but it still needs to point at the template. + tb = fake_traceback( + exc_value, None, exc_value.filename or "", exc_value.lineno + ) + else: + # Skip the frame for the render function. + tb = tb.tb_next + + stack = [] + + # Build the stack of traceback object, replacing any in template + # code with the source file and line information. + while tb is not None: + # Skip frames decorated with @internalcode. These are internal + # calls that aren't useful in template debugging output. + if tb.tb_frame.f_code in internal_code: + tb = tb.tb_next + continue + + template = tb.tb_frame.f_globals.get("__jinja_template__") + + if template is not None: + lineno = template.get_corresponding_lineno(tb.tb_lineno) + fake_tb = fake_traceback(exc_value, tb, template.filename, lineno) + stack.append(fake_tb) + else: + stack.append(tb) + + tb = tb.tb_next + + tb_next = None + + # Assign tb_next in reverse to avoid circular references. + for tb in reversed(stack): + tb_next = tb_set_next(tb, tb_next) + + return exc_type, exc_value, tb_next + + +def fake_traceback(exc_value, tb, filename, lineno): + """Produce a new traceback object that looks like it came from the + template source instead of the compiled code. The filename, line + number, and location name will point to the template, and the local + variables will be the current template context. + + :param exc_value: The original exception to be re-raised to create + the new traceback. + :param tb: The original traceback to get the local variables and + code info from. + :param filename: The template filename. + :param lineno: The line number in the template source. + """ + if tb is not None: + # Replace the real locals with the context that would be + # available at that point in the template. + locals = get_template_locals(tb.tb_frame.f_locals) + locals.pop("__jinja_exception__", None) + else: + locals = {} + + globals = { + "__name__": filename, + "__file__": filename, + "__jinja_exception__": exc_value, + } + # Raise an exception at the correct line number. + code = compile("\n" * (lineno - 1) + "raise __jinja_exception__", filename, "exec") + + # Build a new code object that points to the template file and + # replaces the location with a block name. + try: + location = "template" + + if tb is not None: + function = tb.tb_frame.f_code.co_name + + if function == "root": + location = "top-level template code" + elif function.startswith("block_"): + location = 'block "%s"' % function[6:] + + # Collect arguments for the new code object. CodeType only + # accepts positional arguments, and arguments were inserted in + # new Python versions. + code_args = [] + + for attr in ( + "argcount", + "posonlyargcount", # Python 3.8 + "kwonlyargcount", # Python 3 + "nlocals", + "stacksize", + "flags", + "code", # codestring + "consts", # constants + "names", + "varnames", + ("filename", filename), + ("name", location), + "firstlineno", + "lnotab", + "freevars", + "cellvars", + ): + if isinstance(attr, tuple): + # Replace with given value. + code_args.append(attr[1]) + continue + + try: + # Copy original value if it exists. + code_args.append(getattr(code, "co_" + attr)) + except AttributeError: + # Some arguments were added later. + continue + + code = CodeType(*code_args) + except Exception: + # Some environments such as Google App Engine don't support + # modifying code objects. + pass + + # Execute the new code, which is guaranteed to raise, and return + # the new traceback without this frame. + try: + exec(code, globals, locals) + except BaseException: + return sys.exc_info()[2].tb_next + + +def get_template_locals(real_locals): + """Based on the runtime locals, get the context that would be + available at that point in the template. + """ + # Start with the current template context. + ctx = real_locals.get("context") + + if ctx: + data = ctx.get_all().copy() + else: + data = {} + + # Might be in a derived context that only sets local variables + # rather than pushing a context. Local variables follow the scheme + # l_depth_name. Find the highest-depth local that has a value for + # each name. + local_overrides = {} + + for name, value in real_locals.items(): + if not name.startswith("l_") or value is missing: + # Not a template variable, or no longer relevant. + continue + + try: + _, depth, name = name.split("_", 2) + depth = int(depth) + except ValueError: + continue + + cur_depth = local_overrides.get(name, (-1,))[0] + + if cur_depth < depth: + local_overrides[name] = (depth, value) + + # Modify the context with any derived context. + for name, (_, value) in local_overrides.items(): + if value is missing: + data.pop(name, None) + else: + data[name] = value + + return data + + +if sys.version_info >= (3, 7): + # tb_next is directly assignable as of Python 3.7 + def tb_set_next(tb, tb_next): + tb.tb_next = tb_next + return tb + + +elif PYPY: + # PyPy might have special support, and won't work with ctypes. + try: + import tputil + except ImportError: + # Without tproxy support, use the original traceback. + def tb_set_next(tb, tb_next): + return tb + + else: + # With tproxy support, create a proxy around the traceback that + # returns the new tb_next. + def tb_set_next(tb, tb_next): + def controller(op): + if op.opname == "__getattribute__" and op.args[0] == "tb_next": + return tb_next + + return op.delegate() + + return tputil.make_proxy(controller, obj=tb) + + +else: + # Use ctypes to assign tb_next at the C level since it's read-only + # from Python. + import ctypes + + class _CTraceback(ctypes.Structure): + _fields_ = [ + # Extra PyObject slots when compiled with Py_TRACE_REFS. + ("PyObject_HEAD", ctypes.c_byte * object().__sizeof__()), + # Only care about tb_next as an object, not a traceback. + ("tb_next", ctypes.py_object), + ] + + def tb_set_next(tb, tb_next): + c_tb = _CTraceback.from_address(id(tb)) + + # Clear out the old tb_next. + if tb.tb_next is not None: + c_tb_next = ctypes.py_object(tb.tb_next) + c_tb.tb_next = ctypes.py_object() + ctypes.pythonapi.Py_DecRef(c_tb_next) + + # Assign the new tb_next. + if tb_next is not None: + c_tb_next = ctypes.py_object(tb_next) + ctypes.pythonapi.Py_IncRef(c_tb_next) + c_tb.tb_next = c_tb_next + + return tb diff --git a/magma/squid_cnf/charms/squid/venv/jinja2/defaults.py b/magma/squid_cnf/charms/squid/venv/jinja2/defaults.py new file mode 100644 index 0000000000000000000000000000000000000000..8e0e7d771076a8fe0dc71584018338d2caa35fb5 --- /dev/null +++ b/magma/squid_cnf/charms/squid/venv/jinja2/defaults.py @@ -0,0 +1,44 @@ +# -*- coding: utf-8 -*- +from ._compat import range_type +from .filters import FILTERS as DEFAULT_FILTERS # noqa: F401 +from .tests import TESTS as DEFAULT_TESTS # noqa: F401 +from .utils import Cycler +from .utils import generate_lorem_ipsum +from .utils import Joiner +from .utils import Namespace + +# defaults for the parser / lexer +BLOCK_START_STRING = "{%" +BLOCK_END_STRING = "%}" +VARIABLE_START_STRING = "{{" +VARIABLE_END_STRING = "}}" +COMMENT_START_STRING = "{#" +COMMENT_END_STRING = "#}" +LINE_STATEMENT_PREFIX = None +LINE_COMMENT_PREFIX = None +TRIM_BLOCKS = False +LSTRIP_BLOCKS = False +NEWLINE_SEQUENCE = "\n" +KEEP_TRAILING_NEWLINE = False + +# default filters, tests and namespace + +DEFAULT_NAMESPACE = { + "range": range_type, + "dict": dict, + "lipsum": generate_lorem_ipsum, + "cycler": Cycler, + "joiner": Joiner, + "namespace": Namespace, +} + +# default policies +DEFAULT_POLICIES = { + "compiler.ascii_str": True, + "urlize.rel": "noopener", + "urlize.target": None, + "truncate.leeway": 5, + "json.dumps_function": None, + "json.dumps_kwargs": {"sort_keys": True}, + "ext.i18n.trimmed": False, +} diff --git a/magma/squid_cnf/charms/squid/venv/jinja2/environment.py b/magma/squid_cnf/charms/squid/venv/jinja2/environment.py new file mode 100644 index 0000000000000000000000000000000000000000..8430390eeab412b7f3b92d88a4bdecbe1de5f538 --- /dev/null +++ b/magma/squid_cnf/charms/squid/venv/jinja2/environment.py @@ -0,0 +1,1362 @@ +# -*- coding: utf-8 -*- +"""Classes for managing templates and their runtime and compile time +options. +""" +import os +import sys +import weakref +from functools import partial +from functools import reduce + +from markupsafe import Markup + +from . import nodes +from ._compat import encode_filename +from ._compat import implements_iterator +from ._compat import implements_to_string +from ._compat import iteritems +from ._compat import PY2 +from ._compat import PYPY +from ._compat import reraise +from ._compat import string_types +from ._compat import text_type +from .compiler import CodeGenerator +from .compiler import generate +from .defaults import BLOCK_END_STRING +from .defaults import BLOCK_START_STRING +from .defaults import COMMENT_END_STRING +from .defaults import COMMENT_START_STRING +from .defaults import DEFAULT_FILTERS +from .defaults import DEFAULT_NAMESPACE +from .defaults import DEFAULT_POLICIES +from .defaults import DEFAULT_TESTS +from .defaults import KEEP_TRAILING_NEWLINE +from .defaults import LINE_COMMENT_PREFIX +from .defaults import LINE_STATEMENT_PREFIX +from .defaults import LSTRIP_BLOCKS +from .defaults import NEWLINE_SEQUENCE +from .defaults import TRIM_BLOCKS +from .defaults import VARIABLE_END_STRING +from .defaults import VARIABLE_START_STRING +from .exceptions import TemplateNotFound +from .exceptions import TemplateRuntimeError +from .exceptions import TemplatesNotFound +from .exceptions import TemplateSyntaxError +from .exceptions import UndefinedError +from .lexer import get_lexer +from .lexer import TokenStream +from .nodes import EvalContext +from .parser import Parser +from .runtime import Context +from .runtime import new_context +from .runtime import Undefined +from .utils import concat +from .utils import consume +from .utils import have_async_gen +from .utils import import_string +from .utils import internalcode +from .utils import LRUCache +from .utils import missing + +# for direct template usage we have up to ten living environments +_spontaneous_environments = LRUCache(10) + + +def get_spontaneous_environment(cls, *args): + """Return a new spontaneous environment. A spontaneous environment + is used for templates created directly rather than through an + existing environment. + + :param cls: Environment class to create. + :param args: Positional arguments passed to environment. + """ + key = (cls, args) + + try: + return _spontaneous_environments[key] + except KeyError: + _spontaneous_environments[key] = env = cls(*args) + env.shared = True + return env + + +def create_cache(size): + """Return the cache class for the given size.""" + if size == 0: + return None + if size < 0: + return {} + return LRUCache(size) + + +def copy_cache(cache): + """Create an empty copy of the given cache.""" + if cache is None: + return None + elif type(cache) is dict: + return {} + return LRUCache(cache.capacity) + + +def load_extensions(environment, extensions): + """Load the extensions from the list and bind it to the environment. + Returns a dict of instantiated environments. + """ + result = {} + for extension in extensions: + if isinstance(extension, string_types): + extension = import_string(extension) + result[extension.identifier] = extension(environment) + return result + + +def fail_for_missing_callable(string, name): + msg = string % name + if isinstance(name, Undefined): + try: + name._fail_with_undefined_error() + except Exception as e: + msg = "%s (%s; did you forget to quote the callable name?)" % (msg, e) + raise TemplateRuntimeError(msg) + + +def _environment_sanity_check(environment): + """Perform a sanity check on the environment.""" + assert issubclass( + environment.undefined, Undefined + ), "undefined must be a subclass of undefined because filters depend on it." + assert ( + environment.block_start_string + != environment.variable_start_string + != environment.comment_start_string + ), "block, variable and comment start strings must be different" + assert environment.newline_sequence in ( + "\r", + "\r\n", + "\n", + ), "newline_sequence set to unknown line ending string." + return environment + + +class Environment(object): + r"""The core component of Jinja is the `Environment`. It contains + important shared variables like configuration, filters, tests, + globals and others. Instances of this class may be modified if + they are not shared and if no template was loaded so far. + Modifications on environments after the first template was loaded + will lead to surprising effects and undefined behavior. + + Here are the possible initialization parameters: + + `block_start_string` + The string marking the beginning of a block. Defaults to ``'{%'``. + + `block_end_string` + The string marking the end of a block. Defaults to ``'%}'``. + + `variable_start_string` + The string marking the beginning of a print statement. + Defaults to ``'{{'``. + + `variable_end_string` + The string marking the end of a print statement. Defaults to + ``'}}'``. + + `comment_start_string` + The string marking the beginning of a comment. Defaults to ``'{#'``. + + `comment_end_string` + The string marking the end of a comment. Defaults to ``'#}'``. + + `line_statement_prefix` + If given and a string, this will be used as prefix for line based + statements. See also :ref:`line-statements`. + + `line_comment_prefix` + If given and a string, this will be used as prefix for line based + comments. See also :ref:`line-statements`. + + .. versionadded:: 2.2 + + `trim_blocks` + If this is set to ``True`` the first newline after a block is + removed (block, not variable tag!). Defaults to `False`. + + `lstrip_blocks` + If this is set to ``True`` leading spaces and tabs are stripped + from the start of a line to a block. Defaults to `False`. + + `newline_sequence` + The sequence that starts a newline. Must be one of ``'\r'``, + ``'\n'`` or ``'\r\n'``. The default is ``'\n'`` which is a + useful default for Linux and OS X systems as well as web + applications. + + `keep_trailing_newline` + Preserve the trailing newline when rendering templates. + The default is ``False``, which causes a single newline, + if present, to be stripped from the end of the template. + + .. versionadded:: 2.7 + + `extensions` + List of Jinja extensions to use. This can either be import paths + as strings or extension classes. For more information have a + look at :ref:`the extensions documentation `. + + `optimized` + should the optimizer be enabled? Default is ``True``. + + `undefined` + :class:`Undefined` or a subclass of it that is used to represent + undefined values in the template. + + `finalize` + A callable that can be used to process the result of a variable + expression before it is output. For example one can convert + ``None`` implicitly into an empty string here. + + `autoescape` + If set to ``True`` the XML/HTML autoescaping feature is enabled by + default. For more details about autoescaping see + :class:`~markupsafe.Markup`. As of Jinja 2.4 this can also + be a callable that is passed the template name and has to + return ``True`` or ``False`` depending on autoescape should be + enabled by default. + + .. versionchanged:: 2.4 + `autoescape` can now be a function + + `loader` + The template loader for this environment. + + `cache_size` + The size of the cache. Per default this is ``400`` which means + that if more than 400 templates are loaded the loader will clean + out the least recently used template. If the cache size is set to + ``0`` templates are recompiled all the time, if the cache size is + ``-1`` the cache will not be cleaned. + + .. versionchanged:: 2.8 + The cache size was increased to 400 from a low 50. + + `auto_reload` + Some loaders load templates from locations where the template + sources may change (ie: file system or database). If + ``auto_reload`` is set to ``True`` (default) every time a template is + requested the loader checks if the source changed and if yes, it + will reload the template. For higher performance it's possible to + disable that. + + `bytecode_cache` + If set to a bytecode cache object, this object will provide a + cache for the internal Jinja bytecode so that templates don't + have to be parsed if they were not changed. + + See :ref:`bytecode-cache` for more information. + + `enable_async` + If set to true this enables async template execution which allows + you to take advantage of newer Python features. This requires + Python 3.6 or later. + """ + + #: if this environment is sandboxed. Modifying this variable won't make + #: the environment sandboxed though. For a real sandboxed environment + #: have a look at jinja2.sandbox. This flag alone controls the code + #: generation by the compiler. + sandboxed = False + + #: True if the environment is just an overlay + overlayed = False + + #: the environment this environment is linked to if it is an overlay + linked_to = None + + #: shared environments have this set to `True`. A shared environment + #: must not be modified + shared = False + + #: the class that is used for code generation. See + #: :class:`~jinja2.compiler.CodeGenerator` for more information. + code_generator_class = CodeGenerator + + #: the context class thatis used for templates. See + #: :class:`~jinja2.runtime.Context` for more information. + context_class = Context + + def __init__( + self, + block_start_string=BLOCK_START_STRING, + block_end_string=BLOCK_END_STRING, + variable_start_string=VARIABLE_START_STRING, + variable_end_string=VARIABLE_END_STRING, + comment_start_string=COMMENT_START_STRING, + comment_end_string=COMMENT_END_STRING, + line_statement_prefix=LINE_STATEMENT_PREFIX, + line_comment_prefix=LINE_COMMENT_PREFIX, + trim_blocks=TRIM_BLOCKS, + lstrip_blocks=LSTRIP_BLOCKS, + newline_sequence=NEWLINE_SEQUENCE, + keep_trailing_newline=KEEP_TRAILING_NEWLINE, + extensions=(), + optimized=True, + undefined=Undefined, + finalize=None, + autoescape=False, + loader=None, + cache_size=400, + auto_reload=True, + bytecode_cache=None, + enable_async=False, + ): + # !!Important notice!! + # The constructor accepts quite a few arguments that should be + # passed by keyword rather than position. However it's important to + # not change the order of arguments because it's used at least + # internally in those cases: + # - spontaneous environments (i18n extension and Template) + # - unittests + # If parameter changes are required only add parameters at the end + # and don't change the arguments (or the defaults!) of the arguments + # existing already. + + # lexer / parser information + self.block_start_string = block_start_string + self.block_end_string = block_end_string + self.variable_start_string = variable_start_string + self.variable_end_string = variable_end_string + self.comment_start_string = comment_start_string + self.comment_end_string = comment_end_string + self.line_statement_prefix = line_statement_prefix + self.line_comment_prefix = line_comment_prefix + self.trim_blocks = trim_blocks + self.lstrip_blocks = lstrip_blocks + self.newline_sequence = newline_sequence + self.keep_trailing_newline = keep_trailing_newline + + # runtime information + self.undefined = undefined + self.optimized = optimized + self.finalize = finalize + self.autoescape = autoescape + + # defaults + self.filters = DEFAULT_FILTERS.copy() + self.tests = DEFAULT_TESTS.copy() + self.globals = DEFAULT_NAMESPACE.copy() + + # set the loader provided + self.loader = loader + self.cache = create_cache(cache_size) + self.bytecode_cache = bytecode_cache + self.auto_reload = auto_reload + + # configurable policies + self.policies = DEFAULT_POLICIES.copy() + + # load extensions + self.extensions = load_extensions(self, extensions) + + self.enable_async = enable_async + self.is_async = self.enable_async and have_async_gen + if self.is_async: + # runs patch_all() to enable async support + from . import asyncsupport # noqa: F401 + + _environment_sanity_check(self) + + def add_extension(self, extension): + """Adds an extension after the environment was created. + + .. versionadded:: 2.5 + """ + self.extensions.update(load_extensions(self, [extension])) + + def extend(self, **attributes): + """Add the items to the instance of the environment if they do not exist + yet. This is used by :ref:`extensions ` to register + callbacks and configuration values without breaking inheritance. + """ + for key, value in iteritems(attributes): + if not hasattr(self, key): + setattr(self, key, value) + + def overlay( + self, + block_start_string=missing, + block_end_string=missing, + variable_start_string=missing, + variable_end_string=missing, + comment_start_string=missing, + comment_end_string=missing, + line_statement_prefix=missing, + line_comment_prefix=missing, + trim_blocks=missing, + lstrip_blocks=missing, + extensions=missing, + optimized=missing, + undefined=missing, + finalize=missing, + autoescape=missing, + loader=missing, + cache_size=missing, + auto_reload=missing, + bytecode_cache=missing, + ): + """Create a new overlay environment that shares all the data with the + current environment except for cache and the overridden attributes. + Extensions cannot be removed for an overlayed environment. An overlayed + environment automatically gets all the extensions of the environment it + is linked to plus optional extra extensions. + + Creating overlays should happen after the initial environment was set + up completely. Not all attributes are truly linked, some are just + copied over so modifications on the original environment may not shine + through. + """ + args = dict(locals()) + del args["self"], args["cache_size"], args["extensions"] + + rv = object.__new__(self.__class__) + rv.__dict__.update(self.__dict__) + rv.overlayed = True + rv.linked_to = self + + for key, value in iteritems(args): + if value is not missing: + setattr(rv, key, value) + + if cache_size is not missing: + rv.cache = create_cache(cache_size) + else: + rv.cache = copy_cache(self.cache) + + rv.extensions = {} + for key, value in iteritems(self.extensions): + rv.extensions[key] = value.bind(rv) + if extensions is not missing: + rv.extensions.update(load_extensions(rv, extensions)) + + return _environment_sanity_check(rv) + + lexer = property(get_lexer, doc="The lexer for this environment.") + + def iter_extensions(self): + """Iterates over the extensions by priority.""" + return iter(sorted(self.extensions.values(), key=lambda x: x.priority)) + + def getitem(self, obj, argument): + """Get an item or attribute of an object but prefer the item.""" + try: + return obj[argument] + except (AttributeError, TypeError, LookupError): + if isinstance(argument, string_types): + try: + attr = str(argument) + except Exception: + pass + else: + try: + return getattr(obj, attr) + except AttributeError: + pass + return self.undefined(obj=obj, name=argument) + + def getattr(self, obj, attribute): + """Get an item or attribute of an object but prefer the attribute. + Unlike :meth:`getitem` the attribute *must* be a bytestring. + """ + try: + return getattr(obj, attribute) + except AttributeError: + pass + try: + return obj[attribute] + except (TypeError, LookupError, AttributeError): + return self.undefined(obj=obj, name=attribute) + + def call_filter( + self, name, value, args=None, kwargs=None, context=None, eval_ctx=None + ): + """Invokes a filter on a value the same way the compiler does it. + + Note that on Python 3 this might return a coroutine in case the + filter is running from an environment in async mode and the filter + supports async execution. It's your responsibility to await this + if needed. + + .. versionadded:: 2.7 + """ + func = self.filters.get(name) + if func is None: + fail_for_missing_callable("no filter named %r", name) + args = [value] + list(args or ()) + if getattr(func, "contextfilter", False) is True: + if context is None: + raise TemplateRuntimeError( + "Attempted to invoke context filter without context" + ) + args.insert(0, context) + elif getattr(func, "evalcontextfilter", False) is True: + if eval_ctx is None: + if context is not None: + eval_ctx = context.eval_ctx + else: + eval_ctx = EvalContext(self) + args.insert(0, eval_ctx) + elif getattr(func, "environmentfilter", False) is True: + args.insert(0, self) + return func(*args, **(kwargs or {})) + + def call_test(self, name, value, args=None, kwargs=None): + """Invokes a test on a value the same way the compiler does it. + + .. versionadded:: 2.7 + """ + func = self.tests.get(name) + if func is None: + fail_for_missing_callable("no test named %r", name) + return func(value, *(args or ()), **(kwargs or {})) + + @internalcode + def parse(self, source, name=None, filename=None): + """Parse the sourcecode and return the abstract syntax tree. This + tree of nodes is used by the compiler to convert the template into + executable source- or bytecode. This is useful for debugging or to + extract information from templates. + + If you are :ref:`developing Jinja extensions ` + this gives you a good overview of the node tree generated. + """ + try: + return self._parse(source, name, filename) + except TemplateSyntaxError: + self.handle_exception(source=source) + + def _parse(self, source, name, filename): + """Internal parsing function used by `parse` and `compile`.""" + return Parser(self, source, name, encode_filename(filename)).parse() + + def lex(self, source, name=None, filename=None): + """Lex the given sourcecode and return a generator that yields + tokens as tuples in the form ``(lineno, token_type, value)``. + This can be useful for :ref:`extension development ` + and debugging templates. + + This does not perform preprocessing. If you want the preprocessing + of the extensions to be applied you have to filter source through + the :meth:`preprocess` method. + """ + source = text_type(source) + try: + return self.lexer.tokeniter(source, name, filename) + except TemplateSyntaxError: + self.handle_exception(source=source) + + def preprocess(self, source, name=None, filename=None): + """Preprocesses the source with all extensions. This is automatically + called for all parsing and compiling methods but *not* for :meth:`lex` + because there you usually only want the actual source tokenized. + """ + return reduce( + lambda s, e: e.preprocess(s, name, filename), + self.iter_extensions(), + text_type(source), + ) + + def _tokenize(self, source, name, filename=None, state=None): + """Called by the parser to do the preprocessing and filtering + for all the extensions. Returns a :class:`~jinja2.lexer.TokenStream`. + """ + source = self.preprocess(source, name, filename) + stream = self.lexer.tokenize(source, name, filename, state) + for ext in self.iter_extensions(): + stream = ext.filter_stream(stream) + if not isinstance(stream, TokenStream): + stream = TokenStream(stream, name, filename) + return stream + + def _generate(self, source, name, filename, defer_init=False): + """Internal hook that can be overridden to hook a different generate + method in. + + .. versionadded:: 2.5 + """ + return generate( + source, + self, + name, + filename, + defer_init=defer_init, + optimized=self.optimized, + ) + + def _compile(self, source, filename): + """Internal hook that can be overridden to hook a different compile + method in. + + .. versionadded:: 2.5 + """ + return compile(source, filename, "exec") + + @internalcode + def compile(self, source, name=None, filename=None, raw=False, defer_init=False): + """Compile a node or template source code. The `name` parameter is + the load name of the template after it was joined using + :meth:`join_path` if necessary, not the filename on the file system. + the `filename` parameter is the estimated filename of the template on + the file system. If the template came from a database or memory this + can be omitted. + + The return value of this method is a python code object. If the `raw` + parameter is `True` the return value will be a string with python + code equivalent to the bytecode returned otherwise. This method is + mainly used internally. + + `defer_init` is use internally to aid the module code generator. This + causes the generated code to be able to import without the global + environment variable to be set. + + .. versionadded:: 2.4 + `defer_init` parameter added. + """ + source_hint = None + try: + if isinstance(source, string_types): + source_hint = source + source = self._parse(source, name, filename) + source = self._generate(source, name, filename, defer_init=defer_init) + if raw: + return source + if filename is None: + filename = "