X-Git-Url: https://osm.etsi.org/gitweb/?a=blobdiff_plain;f=juju%2Fclient%2Ffacade.py;h=c959e01649964a9a5eb955e2fc0829ebabfcc90f;hb=17b26ef759a99c9010ea30e47205bfb332400e74;hp=7b7d9c70240642ded9c34da713b38ff2e9d7db49;hpb=c2ca299be7f2663544d4d3ebd85ee19b5011b190;p=osm%2FN2VC.git diff --git a/juju/client/facade.py b/juju/client/facade.py index 7b7d9c7..c959e01 100644 --- a/juju/client/facade.py +++ b/juju/client/facade.py @@ -1,18 +1,27 @@ import argparse import builtins +from collections import defaultdict import functools +from glob import glob import json import keyword from pathlib import Path import pprint +import re import textwrap -from typing import Sequence, Mapping, TypeVar, Any, Union, Optional +from typing import Sequence, Mapping, TypeVar, Any, Union import typing from . import codegen _marker = object() +JUJU_VERSION = re.compile('[0-9]+\.[0-9-]+[\.\-][0-9a-z]+(\.[0-9]+)?') +# Workaround for https://bugs.launchpad.net/juju/+bug/1683906 +NAUGHTY_CLASSES = ['ClientFacade', 'Client', 'FullStatus', 'ModelStatusInfo', + 'ModelInfo'] + + # Map basic types to Python's typing with a callable SCHEMA_TO_PYTHON = { 'string': str, @@ -24,6 +33,66 @@ SCHEMA_TO_PYTHON = { } +# Friendly warning message to stick at the top of generated files. +HEADER = """\ +# DO NOT CHANGE THIS FILE! This file is auto-generated by facade.py. +# Changes will be overwritten/lost when the file is regenerated. + +""" + + +# Classes and helper functions that we'll write to _client.py +LOOKUP_FACADE = ''' +def lookup_facade(name, version): + """ + Given a facade name and version, attempt to pull that facade out + of the correct client.py file. + + """ + try: + facade = getattr(CLIENTS[str(version)], name) + except KeyError: + raise ImportError("No facades found for version {}".format(version)) + except AttributeError: + raise ImportError( + "No facade with name '{}' in version {}".format(name, version)) + return facade + + +''' + +TYPE_FACTORY = ''' +class TypeFactory: + @classmethod + def from_connection(cls, connection): + """ + Given a connected Connection object, return an initialized and + connected instance of an API Interface matching the name of + this class. + + @param connection: initialized Connection object. + + """ + version = connection.facades[cls.__name__[:-6]] + + c = lookup_facade(cls.__name__, version) + c = c() + c.connect(connection) + + return c + + +''' + +CLIENT_TABLE = ''' +CLIENTS = {{ + {clients} +}} + + +''' + + class KindRegistry(dict): def register(self, name, version, obj): self[name] = {version: { @@ -60,7 +129,9 @@ class TypeRegistry(dict): _types = TypeRegistry() _registry = KindRegistry() -classes = {} +CLASSES = {} +factories = codegen.Capture() + def booler(v): if isinstance(v, str): @@ -101,6 +172,8 @@ def name_to_py(name): def strcast(kind, keep_builtins=False): if issubclass(kind, typing.GenericMeta): return str(kind)[1:] + if str(kind).startswith('~'): + return str(kind)[1:] if (kind in basic_types or type(kind) in basic_types) and keep_builtins is False: return kind.__name__ @@ -110,7 +183,6 @@ def strcast(kind, keep_builtins=False): class Args(list): def __init__(self, defs): self.defs = defs - #self.append("self") if defs: rtypes = _registry.getObj(_types[defs]) if len(rtypes) == 1: @@ -122,7 +194,7 @@ class Args(list): self.append((name, rtype)) def do_explode(self, kind): - if kind in basic_types: + if kind in basic_types or type(kind) is typing.TypeVar: return False if not issubclass(kind, (typing.Sequence, typing.Mapping)): @@ -162,6 +234,14 @@ class Args(list): return parts return '' + def as_kwargs(self): + if self: + parts = [] + for item in self: + parts.append('{}=None'.format(name_to_py(item[0]))) + return ', '.join(parts) + return '' + def typed(self): return self._get_arg_str(True) @@ -173,42 +253,80 @@ class Args(list): def buildTypes(schema, capture): - global classes INDENT = " " for kind in sorted((k for k in _types if not isinstance(k, str)), key=lambda x: str(x)): name = _types[kind] - args = Args(kind) - if name in classes: + if name in capture and not name in NAUGHTY_CLASSES: continue + args = Args(kind) + # Write Factory class for _client.py + make_factory(name) + # Write actual class source = [""" class {}(Type): _toSchema = {} _toPy = {} - def __init__(self{}{}): + def __init__(self{}{}, **unknown_fields): ''' {} - '''""".format(name, - args.PyToSchemaMapping(), - args.SchemaToPyMapping(), - ", " if args else "", - args, - textwrap.indent(args.get_doc(), INDENT *2)) - #pprint.pformat(schema['definitions'][name])) + '''""".format( + name, + # pprint these to get stable ordering across regens + pprint.pformat(args.PyToSchemaMapping(), width=999), + pprint.pformat(args.SchemaToPyMapping(), width=999), + ", " if args else "", + args.as_kwargs(), + textwrap.indent(args.get_doc(), INDENT * 2)) ] assignments = args._get_arg_str(False, False) - for assign in assignments: - source.append("{}self.{} = {}".format(INDENT * 2, assign, assign)) - if not assignments: - source.append("{}pass".format(INDENT *2)) + + if not args: + source.append("{}pass".format(INDENT * 2)) + else: + for arg in args: + arg_name = name_to_py(arg[0]) + arg_type = arg[1] + arg_type_name = strcast(arg_type) + if arg_type in basic_types: + source.append("{}self.{} = {}".format(INDENT * 2, arg_name, arg_name)) + elif issubclass(arg_type, typing.Sequence): + value_type = ( + arg_type_name.__parameters__[0] + if len(arg_type_name.__parameters__) + else None + ) + if type(value_type) is typing.TypeVar: + source.append("{}self.{} = [{}.from_json(o) for o in {} or []]".format( + INDENT * 2, arg_name, strcast(value_type), arg_name)) + else: + source.append("{}self.{} = {}".format(INDENT * 2, arg_name, arg_name)) + elif issubclass(arg_type, typing.Mapping): + value_type = ( + arg_type_name.__parameters__[1] + if len(arg_type_name.__parameters__) > 1 + else None + ) + if type(value_type) is typing.TypeVar: + source.append("{}self.{} = {{k: {}.from_json(v) for k, v in ({} or dict()).items()}}".format( + INDENT * 2, arg_name, strcast(value_type), arg_name)) + else: + source.append("{}self.{} = {}".format(INDENT * 2, arg_name, arg_name)) + elif type(arg_type) is typing.TypeVar: + source.append("{}self.{} = {}.from_json({}) if {} else None".format( + INDENT * 2, arg_name, arg_type_name, arg_name, arg_name)) + else: + source.append("{}self.{} = {}".format(INDENT * 2, arg_name, arg_name)) + source = "\n".join(source) - capture.write(source) - capture.write("\n\n") + capture.clear(name) + capture[name].write(source) + capture[name].write("\n\n") co = compile(source, __name__, "exec") ns = _getns() exec(co, ns) cls = ns[name] - classes[name] = cls + CLASSES[name] = cls def retspec(defs): @@ -219,6 +337,8 @@ def retspec(defs): # Error or the expected Type if not defs: return None + if defs in basic_types: + return strcast(defs, False) rtypes = _registry.getObj(_types[defs]) if not rtypes: return None @@ -267,18 +387,27 @@ def ReturnMapping(cls): # so the value can be cast def decorator(f): @functools.wraps(f) - def wrapper(*args, **kwargs): - reply = f(*args, **kwargs) - if cls is None or reply: + async def wrapper(*args, **kwargs): + nonlocal cls + reply = await f(*args, **kwargs) + if cls is None: return reply - if 'Error' in reply: - cls = Error + if 'error' in reply: + cls = CLASSES['Error'] if issubclass(cls, typing.Sequence): result = [] + item_cls = cls.__parameters__[0] for item in reply: + result.append(item_cls.from_json(item)) + """ + if 'error' in item: + cls = CLASSES['Error'] + else: + cls = item_cls result.append(cls.from_json(item)) + """ else: - result = cls.from_json(reply) + result = cls.from_json(reply['response']) return result return wrapper @@ -291,25 +420,25 @@ def makeFunc(cls, name, params, result, async=True): assignments = [] toschema = args.PyToSchemaMapping() for arg in args._get_arg_str(False, False): - assignments.append("{}params[\'{}\'] = {}".format(INDENT, - toschema[arg], - arg)) + assignments.append("{}_params[\'{}\'] = {}".format(INDENT, + toschema[arg], + arg)) assignments = "\n".join(assignments) res = retspec(result) source = """ -#@ReturnMapping({rettype}) +@ReturnMapping({rettype}) {async}def {name}(self{argsep}{args}): ''' {docstring} Returns -> {res} ''' # map input types to rpc msg - params = dict() - msg = dict(Type='{cls.name}', Request='{name}', Version={cls.version}, Params=params) + _params = dict() + msg = dict(type='{cls.name}', request='{name}', version={cls.version}, params=_params) {assignments} reply = {await}self.rpc(msg) - return self._map(reply, {name}) + return reply """ @@ -317,7 +446,6 @@ def makeFunc(cls, name, params, result, async=True): name=name, argsep=", " if args else "", args=args, - #ressep= " -> " if res else "", res=res, rettype=result.__name__ if result else None, docstring=textwrap.indent(args.get_doc(), INDENT), @@ -335,7 +463,7 @@ def buildMethods(cls, capture): for methodname in sorted(properties): method, source = _buildMethod(cls, methodname) setattr(cls, methodname, method) - capture.write(source, depth=1) + capture["{}Facade".format(cls.__name__)].write(source, depth=1) def _buildMethod(cls, name): @@ -349,7 +477,10 @@ def _buildMethod(cls, name): params = _types.get(spec['$ref']) spec = prop.get('Result') if spec: - result = _types.get(spec['$ref']) + if '$ref' in spec: + result = _types.get(spec['$ref']) + else: + result = SCHEMA_TO_PYTHON[spec['type']] return makeFunc(cls, name, params, result) @@ -358,7 +489,7 @@ def buildFacade(schema): version=schema.version, schema=schema)) source = """ -class {name}(Type): +class {name}Facade(Type): name = '{name}' version = {version} schema = {schema} @@ -368,33 +499,38 @@ class {name}(Type): return cls, source +class TypeEncoder(json.JSONEncoder): + def default(self, obj): + if isinstance(obj, Type): + return obj.serialize() + return json.JSONEncoder.default(self, obj) + + class Type: def connect(self, connection): self.connection = connection async def rpc(self, msg): - result = await self.connection.rpc(msg) + result = await self.connection.rpc(msg, encoder=TypeEncoder) return result - def _map(self, reply, method): - # Error, expected return or None - if not reply: - return None - - if 'Error' in reply: - retcls = classes['Error'] - data = reply['Error'] - classes["Error"] - elif 'Response' in reply: - retcls = method.__return_type__ - data = reply['Response'] - return retcls.from_json(data) - @classmethod def from_json(cls, data): + if isinstance(data, cls): + return data if isinstance(data, str): - data = json.loads(data) - return cls(**data) + try: + data = json.loads(data) + except json.JSONDecodeError: + raise + d = {} + for k, v in (data or {}).items(): + d[cls._toPy.get(k, k)] = v + + try: + return cls(**d) + except TypeError: + raise def serialize(self): d = {} @@ -440,7 +576,7 @@ class Schema(dict): if not defs: return for d, data in defs.items(): - if d in _registry: + if d in _registry and not d in NAUGHTY_CLASSES: continue node = self.deref(data, d) kind = node.get("type") @@ -449,6 +585,10 @@ class Schema(dict): elif kind == "array": pass _registry.register(d, self.version, result) + # XXX: This makes sure that the type gets added to the global + # _types dict even if no other type in the schema has a ref + # to it. + getRefType(d) def buildObject(self, node, name=None, d=0): # we don't need to build types recursively here @@ -460,7 +600,10 @@ class Schema(dict): props = node.get("properties") pprops = node.get("patternProperties") if props: - for p, prop in props.items(): + # Sort these so the __init__ arg list for each Type remains + # consistently ordered across regens of client.py + for p in sorted(props): + prop = props[p] if "$ref" in prop: add((p, refType(prop))) else: @@ -474,7 +617,7 @@ class Schema(dict): if pprops: if ".*" not in pprops: raise ValueError( - "Cannot handle actual pattern in patterProperties %s" % + "Cannot handle actual pattern in patternProperties %s" % pprops) pprop = pprops[".*"] if "$ref" in pprop: @@ -485,7 +628,10 @@ class Schema(dict): add((name, self.buildArray(pprop, d + 1))) else: add((name, Mapping[str, SCHEMA_TO_PYTHON[ppkind]])) - #print("{}{}".format(d * " ", struct)) + + if not struct and node.get('additionalProperties', False): + add((name, Mapping[str, SCHEMA_TO_PYTHON['object']])) + return struct def buildArray(self, obj, d=0): @@ -496,7 +642,7 @@ class Schema(dict): kind = obj.get("type") if kind and kind == "array": items = obj['items'] - return self.buildArray(items, d+1) + return self.buildArray(items, d + 1) else: return Sequence[objType(obj)] @@ -512,45 +658,127 @@ def _getns(): return ns - -def generate_facacdes(options): - global classes - schemas = json.loads(Path(options.schema).read_text("utf-8")) - capture = codegen.CodeWriter() - capture.write(""" -from libjuju.facade import Type, ReturnMapping - """) - schemas = [Schema(s) for s in schemas] - - for schema in schemas: - schema.buildDefinitions() - buildTypes(schema, capture) - - for schema in schemas: - # TODO generate class now with a metaclass that takes the schema - # the generated class has the right name and it in turn uses - # the metaclass to populate cls - cls, source = buildFacade(schema) - capture.write(source) - buildMethods(cls, capture) - classes[schema.name] = cls - - return capture +def make_factory(name): + if name in factories: + del factories[name] + factories[name].write("class {}(TypeFactory):\n pass\n\n".format(name)) + + +def write_facades(captures, options): + """ + Write the Facades to the appropriate _client.py + + """ + for version in sorted(captures.keys()): + filename = "{}/_client{}.py".format(options.output_dir, version) + with open(filename, "w") as f: + f.write(HEADER) + f.write("from juju.client.facade import Type, ReturnMapping\n") + f.write("from juju.client._definitions import *\n\n") + for key in sorted( + [k for k in captures[version].keys() if "Facade" in k]): + print(captures[version][key], file=f) + + # Return the last (most recent) version for use in other routines. + return version + + +def write_definitions(captures, options, version): + """ + Write auxillary (non versioned) classes to + _definitions.py The auxillary classes currently get + written redudantly into each capture object, so we can look in + one of them -- we just use the last one from the loop above. + + """ + with open("{}/_definitions.py".format(options.output_dir), "w") as f: + f.write(HEADER) + f.write("from juju.client.facade import Type, ReturnMapping\n\n") + for key in sorted( + [k for k in captures[version].keys() if "Facade" not in k]): + print(captures[version][key], file=f) + + +def write_client(captures, options): + """ + Write the TypeFactory classes to _client.py, along with some + imports and tables so that we can look up versioned Facades. + + """ + with open("{}/_client.py".format(options.output_dir), "w") as f: + f.write(HEADER) + f.write("from juju.client._definitions import *\n\n") + clients = ", ".join("_client{}".format(v) for v in captures) + f.write("from juju.client import " + clients + "\n\n") + f.write(CLIENT_TABLE.format(clients=",\n ".join( + ['"{}": _client{}'.format(v, v) for v in captures]))) + f.write(LOOKUP_FACADE) + f.write(TYPE_FACTORY) + for key in sorted([k for k in factories.keys() if "Facade" in k]): + print(factories[key], file=f) + + +def generate_facades(options): + captures = defaultdict(codegen.Capture) + schemas = {} + for p in sorted(glob(options.schema)): + if 'latest' in p: + juju_version = 'latest' + else: + try: + juju_version = re.search(JUJU_VERSION, p).group() + except AttributeError: + print("Cannot extract a juju version from {}".format(p)) + print("Schemas must include a juju version in the filename") + raise SystemExit(1) + + new_schemas = json.loads(Path(p).read_text("utf-8")) + schemas[juju_version] = [Schema(s) for s in new_schemas] + + # Build all of the auxillary (unversioned) classes + # TODO: get rid of some of the excess trips through loops in the + # called functions. + for juju_version in sorted(schemas.keys()): + for schema in schemas[juju_version]: + schema.buildDefinitions() + buildTypes(schema, captures[schema.version]) + + # Build the Facade classes + for juju_version in sorted(schemas.keys()): + for schema in schemas[juju_version]: + cls, source = buildFacade(schema) + cls_name = "{}Facade".format(schema.name) + + captures[schema.version].clear(cls_name) + # Make the factory class for _client.py + make_factory(cls_name) + # Make the actual class + captures[schema.version][cls_name].write(source) + # Build the methods for each Facade class. + buildMethods(cls, captures[schema.version]) + # Mark this Facade class as being done for this version -- + # helps mitigate some excessive looping. + CLASSES[schema.name] = cls + + return captures def setup(): parser = argparse.ArgumentParser() - parser.add_argument("-s", "--schema", default="schemas.json") - parser.add_argument("-o", "--output", default="client.py") + parser.add_argument("-s", "--schema", default="juju/client/schemas*") + parser.add_argument("-o", "--output_dir", default="juju/client") options = parser.parse_args() return options def main(): options = setup() - capture = generate_facacdes(options) - with open(options.output, "w") as fp: - print(capture, file=fp) + # Generate some text blobs + captures = generate_facades(options) + # ... and write them out + last_version = write_facades(captures, options) + write_definitions(captures, options, last_version) + write_client(captures, options) if __name__ == '__main__': main()