Feature/api version support (#109)
[osm/N2VC.git] / juju / client / facade.py
1 import argparse
2 import builtins
3 from collections import defaultdict
4 import functools
5 from glob import glob
6 import json
7 import keyword
8 from pathlib import Path
9 import pprint
10 import re
11 import textwrap
12 from typing import Sequence, Mapping, TypeVar, Any, Union
13 import typing
14
15 from . import codegen
16
17 _marker = object()
18
19 JUJU_VERSION = re.compile('[0-9]+\.[0-9-]+[\.\-][0-9a-z]+(\.[0-9]+)?')
20 VERSION_MAP = defaultdict(dict)
21 # Workaround for https://bugs.launchpad.net/juju/+bug/1683906
22 NAUGHTY_CLASSES = ['ClientFacade', 'Client', 'FullStatus', 'ModelStatusInfo',
23 'ModelInfo']
24
25
26 # Map basic types to Python's typing with a callable
27 SCHEMA_TO_PYTHON = {
28 'string': str,
29 'integer': int,
30 'float': float,
31 'number': float,
32 'boolean': bool,
33 'object': Any,
34 }
35
36
37 # Friendly warning message to stick at the top of generated files.
38 HEADER = """\
39 # DO NOT CHANGE THIS FILE! This file is auto-generated by facade.py.
40 # Changes will be overwritten/lost when the file is regenerated.
41
42 """
43
44
45 # Classes and helper functions that we'll write to _client.py
46 LOOKUP_FACADE = '''
47 def lookup_facade(name, version):
48 """
49 Given a facade name and version, attempt to pull that facade out
50 of the correct client<version>.py file.
51
52 """
53 try:
54 facade = getattr(CLIENTS[str(version)], name)
55 except KeyError:
56 raise ImportError("No facades found for version {}".format(version))
57 except AttributeError:
58 raise ImportError(
59 "No facade with name '{}' in version {}".format(name, version))
60 return facade
61
62
63 '''
64
65 TYPE_FACTORY = '''
66 class TypeFactory:
67 @classmethod
68 def from_connection(cls, connection):
69 """
70 Given a connected Connection object, return an initialized and
71 connected instance of an API Interface matching the name of
72 this class.
73
74 @param connection: initialized Connection object.
75
76 """
77 version = connection.facades[cls.__name__[:-6]]
78
79 c = lookup_facade(cls.__name__, version)
80 c = c()
81 c.connect(connection)
82
83 return c
84
85
86 '''
87
88 CLIENT_TABLE = '''
89 CLIENTS = {{
90 {clients}
91 }}
92
93
94 '''
95
96
97 class KindRegistry(dict):
98 def register(self, name, version, obj):
99 self[name] = {version: {
100 "object": obj,
101 }}
102
103 def lookup(self, name, version=None):
104 """If version is omitted, max version is used"""
105 versions = self.get(name)
106 if not versions:
107 return None
108 if version:
109 return versions[version]
110 return versions[max(versions)]
111
112 def getObj(self, name, version=None):
113 result = self.lookup(name, version)
114 if result:
115 obj = result["object"]
116 return obj
117 return None
118
119
120 class TypeRegistry(dict):
121 def get(self, name):
122 # Two way mapping
123 refname = Schema.referenceName(name)
124 if refname not in self:
125 result = TypeVar(refname)
126 self[refname] = result
127 self[result] = refname
128
129 return self[refname]
130
131 _types = TypeRegistry()
132 _registry = KindRegistry()
133 CLASSES = {}
134 factories = codegen.Capture()
135
136
137 def booler(v):
138 if isinstance(v, str):
139 if v == "false":
140 return False
141 return bool(v)
142
143
144 def getRefType(ref):
145 return _types.get(ref)
146
147
148 def refType(obj):
149 return getRefType(obj["$ref"])
150
151
152 def objType(obj):
153 kind = obj.get('type')
154 if not kind:
155 raise ValueError("%s has no type" % obj)
156 result = SCHEMA_TO_PYTHON.get(kind)
157 if not result:
158 raise ValueError("%s has type %s" % (obj, kind))
159 return result
160
161
162 basic_types = [str, bool, int, float]
163
164
165 def name_to_py(name):
166 result = name.replace("-", "_")
167 result = result.lower()
168 if keyword.iskeyword(result) or result in dir(builtins):
169 result += "_"
170 return result
171
172
173 def strcast(kind, keep_builtins=False):
174 if issubclass(kind, typing.GenericMeta):
175 return str(kind)[1:]
176 if str(kind).startswith('~'):
177 return str(kind)[1:]
178 if (kind in basic_types or
179 type(kind) in basic_types) and keep_builtins is False:
180 return kind.__name__
181 return kind
182
183
184 class Args(list):
185 def __init__(self, defs):
186 self.defs = defs
187 if defs:
188 rtypes = _registry.getObj(_types[defs])
189 if len(rtypes) == 1:
190 if not self.do_explode(rtypes[0][1]):
191 for name, rtype in rtypes:
192 self.append((name, rtype))
193 else:
194 for name, rtype in rtypes:
195 self.append((name, rtype))
196
197 def do_explode(self, kind):
198 if kind in basic_types or type(kind) is typing.TypeVar:
199 return False
200 if not issubclass(kind, (typing.Sequence,
201 typing.Mapping)):
202 self.clear()
203 self.extend(Args(kind))
204 return True
205 return False
206
207 def PyToSchemaMapping(self):
208 m = {}
209 for n, rt in self:
210 m[name_to_py(n)] = n
211 return m
212
213 def SchemaToPyMapping(self):
214 m = {}
215 for n, tr in self:
216 m[n] = name_to_py(n)
217 return m
218
219 def _format(self, name, rtype, typed=True):
220 if typed:
221 return "{} : {}".format(
222 name_to_py(name),
223 strcast(rtype)
224 )
225 else:
226 return name_to_py(name)
227
228 def _get_arg_str(self, typed=False, joined=", "):
229 if self:
230 parts = []
231 for item in self:
232 parts.append(self._format(item[0], item[1], typed))
233 if joined:
234 return joined.join(parts)
235 return parts
236 return ''
237
238 def as_kwargs(self):
239 if self:
240 parts = []
241 for item in self:
242 parts.append('{}=None'.format(name_to_py(item[0])))
243 return ', '.join(parts)
244 return ''
245
246 def typed(self):
247 return self._get_arg_str(True)
248
249 def __str__(self):
250 return self._get_arg_str(False)
251
252 def get_doc(self):
253 return self._get_arg_str(True, "\n")
254
255
256 def buildTypes(schema, capture):
257 INDENT = " "
258 for kind in sorted((k for k in _types if not isinstance(k, str)),
259 key=lambda x: str(x)):
260 name = _types[kind]
261 if name in capture and not name in NAUGHTY_CLASSES:
262 continue
263 args = Args(kind)
264 # Write Factory class for _client.py
265 make_factory(name)
266 # Write actual class
267 source = ["""
268 class {}(Type):
269 _toSchema = {}
270 _toPy = {}
271 def __init__(self{}{}):
272 '''
273 {}
274 '''""".format(
275 name,
276 # pprint these to get stable ordering across regens
277 pprint.pformat(args.PyToSchemaMapping(), width=999),
278 pprint.pformat(args.SchemaToPyMapping(), width=999),
279 ", " if args else "",
280 args.as_kwargs(),
281 textwrap.indent(args.get_doc(), INDENT * 2))
282 ]
283 assignments = args._get_arg_str(False, False)
284
285 if not args:
286 source.append("{}pass".format(INDENT * 2))
287 else:
288 for arg in args:
289 arg_name = name_to_py(arg[0])
290 arg_type = arg[1]
291 arg_type_name = strcast(arg_type)
292 if arg_type in basic_types:
293 source.append("{}self.{} = {}".format(INDENT * 2, arg_name, arg_name))
294 elif issubclass(arg_type, typing.Sequence):
295 value_type = (
296 arg_type_name.__parameters__[0]
297 if len(arg_type_name.__parameters__)
298 else None
299 )
300 if type(value_type) is typing.TypeVar:
301 source.append("{}self.{} = [{}.from_json(o) for o in {} or []]".format(
302 INDENT * 2, arg_name, strcast(value_type), arg_name))
303 else:
304 source.append("{}self.{} = {}".format(INDENT * 2, arg_name, arg_name))
305 elif issubclass(arg_type, typing.Mapping):
306 value_type = (
307 arg_type_name.__parameters__[1]
308 if len(arg_type_name.__parameters__) > 1
309 else None
310 )
311 if type(value_type) is typing.TypeVar:
312 source.append("{}self.{} = {{k: {}.from_json(v) for k, v in ({} or dict()).items()}}".format(
313 INDENT * 2, arg_name, strcast(value_type), arg_name))
314 else:
315 source.append("{}self.{} = {}".format(INDENT * 2, arg_name, arg_name))
316 elif type(arg_type) is typing.TypeVar:
317 source.append("{}self.{} = {}.from_json({}) if {} else None".format(
318 INDENT * 2, arg_name, arg_type_name, arg_name, arg_name))
319 else:
320 source.append("{}self.{} = {}".format(INDENT * 2, arg_name, arg_name))
321
322 source = "\n".join(source)
323 capture.clear(name)
324 capture[name].write(source)
325 capture[name].write("\n\n")
326 co = compile(source, __name__, "exec")
327 ns = _getns()
328 exec(co, ns)
329 cls = ns[name]
330 CLASSES[name] = cls
331
332
333 def retspec(defs):
334 # return specs
335 # only return 1, so if there is more than one type
336 # we need to include a union
337 # In truth there is only 1 return
338 # Error or the expected Type
339 if not defs:
340 return None
341 if defs in basic_types:
342 return strcast(defs, False)
343 rtypes = _registry.getObj(_types[defs])
344 if not rtypes:
345 return None
346 if len(rtypes) > 1:
347 return Union[tuple([strcast(r[1], True) for r in rtypes])]
348 return strcast(rtypes[0][1], False)
349
350
351 def return_type(defs):
352 if not defs:
353 return None
354 rtypes = _registry.getObj(_types[defs])
355 if not rtypes:
356 return None
357 if len(rtypes) > 1:
358 for n, t in rtypes:
359 if n == "Error":
360 continue
361 return t
362 return rtypes[0][1]
363
364
365 def type_anno_func(func, defs, is_result=False):
366 annos = {}
367 if not defs:
368 return func
369 rtypes = _registry.getObj(_types[defs])
370 if is_result:
371 kn = "return"
372 if not rtypes:
373 annos[kn] = None
374 elif len(rtypes) > 1:
375 annos[kn] = Union[tuple([r[1] for r in rtypes])]
376 else:
377 annos[kn] = rtypes[0][1]
378 else:
379 for name, rtype in rtypes:
380 name = name_to_py(name)
381 annos[name] = rtype
382 func.__annotations__.update(annos)
383 return func
384
385
386 def ReturnMapping(cls):
387 # Annotate the method with a return Type
388 # so the value can be cast
389 def decorator(f):
390 @functools.wraps(f)
391 async def wrapper(*args, **kwargs):
392 nonlocal cls
393 reply = await f(*args, **kwargs)
394 if cls is None:
395 return reply
396 if 'error' in reply:
397 cls = CLASSES['Error']
398 if issubclass(cls, typing.Sequence):
399 result = []
400 item_cls = cls.__parameters__[0]
401 for item in reply:
402 result.append(item_cls.from_json(item))
403 """
404 if 'error' in item:
405 cls = CLASSES['Error']
406 else:
407 cls = item_cls
408 result.append(cls.from_json(item))
409 """
410 else:
411 result = cls.from_json(reply['response'])
412
413 return result
414 return wrapper
415 return decorator
416
417
418 def makeFunc(cls, name, params, result, async=True):
419 INDENT = " "
420 args = Args(params)
421 assignments = []
422 toschema = args.PyToSchemaMapping()
423 for arg in args._get_arg_str(False, False):
424 assignments.append("{}_params[\'{}\'] = {}".format(INDENT,
425 toschema[arg],
426 arg))
427 assignments = "\n".join(assignments)
428 res = retspec(result)
429 source = """
430
431 @ReturnMapping({rettype})
432 {async}def {name}(self{argsep}{args}):
433 '''
434 {docstring}
435 Returns -> {res}
436 '''
437 # map input types to rpc msg
438 _params = dict()
439 msg = dict(type='{cls.name}', request='{name}', version={cls.version}, params=_params)
440 {assignments}
441 reply = {await}self.rpc(msg)
442 return reply
443
444 """
445
446 fsource = source.format(async="async " if async else "",
447 name=name,
448 argsep=", " if args else "",
449 args=args,
450 res=res,
451 rettype=result.__name__ if result else None,
452 docstring=textwrap.indent(args.get_doc(), INDENT),
453 cls=cls,
454 assignments=assignments,
455 await="await " if async else "")
456 ns = _getns()
457 exec(fsource, ns)
458 func = ns[name]
459 return func, fsource
460
461
462 def buildMethods(cls, capture):
463 properties = cls.schema['properties']
464 for methodname in sorted(properties):
465 method, source = _buildMethod(cls, methodname)
466 setattr(cls, methodname, method)
467 capture["{}Facade".format(cls.__name__)].write(source, depth=1)
468
469
470 def _buildMethod(cls, name):
471 params = None
472 result = None
473 method = cls.schema['properties'][name]
474 if 'properties' in method:
475 prop = method['properties']
476 spec = prop.get('Params')
477 if spec:
478 params = _types.get(spec['$ref'])
479 spec = prop.get('Result')
480 if spec:
481 if '$ref' in spec:
482 result = _types.get(spec['$ref'])
483 else:
484 result = SCHEMA_TO_PYTHON[spec['type']]
485 return makeFunc(cls, name, params, result)
486
487
488 def buildFacade(schema):
489 cls = type(schema.name, (Type,), dict(name=schema.name,
490 version=schema.version,
491 schema=schema))
492 source = """
493 class {name}Facade(Type):
494 name = '{name}'
495 version = {version}
496 schema = {schema}
497 """.format(name=schema.name,
498 version=schema.version,
499 schema=textwrap.indent(pprint.pformat(schema), " "))
500 return cls, source
501
502
503 class TypeEncoder(json.JSONEncoder):
504 def default(self, obj):
505 if isinstance(obj, Type):
506 return obj.serialize()
507 return json.JSONEncoder.default(self, obj)
508
509
510 class Type:
511 def connect(self, connection):
512 self.connection = connection
513
514 async def rpc(self, msg):
515 result = await self.connection.rpc(msg, encoder=TypeEncoder)
516 return result
517
518 @classmethod
519 def from_json(cls, data):
520 if isinstance(data, cls):
521 return data
522 if isinstance(data, str):
523 data = json.loads(data)
524 d = {}
525 for k, v in (data or {}).items():
526 d[cls._toPy.get(k, k)] = v
527
528 try:
529 return cls(**d)
530 except TypeError:
531 raise
532
533 def serialize(self):
534 d = {}
535 for attr, tgt in self._toSchema.items():
536 d[tgt] = getattr(self, attr)
537 return d
538
539 def to_json(self):
540 return json.dumps(self.serialize())
541
542
543 class Schema(dict):
544 def __init__(self, schema):
545 self.name = schema['Name']
546 self.version = schema['Version']
547 self.update(schema['Schema'])
548
549 @classmethod
550 def referenceName(cls, ref):
551 if ref.startswith("#/definitions/"):
552 ref = ref.rsplit("/", 1)[-1]
553 return ref
554
555 def resolveDefinition(self, ref):
556 return self['definitions'][self.referenceName(ref)]
557
558 def deref(self, prop, name):
559 if not isinstance(prop, dict):
560 raise TypeError(prop)
561 if "$ref" not in prop:
562 return prop
563
564 target = self.resolveDefinition(prop["$ref"])
565 return target
566
567 def buildDefinitions(self):
568 # here we are building the types out
569 # anything in definitions is a type
570 # but these may contain references themselves
571 # so we dfs to the bottom and build upwards
572 # when a types is already in the registry
573 defs = self.get('definitions')
574 if not defs:
575 return
576 for d, data in defs.items():
577 if d in _registry and not d in NAUGHTY_CLASSES:
578 continue
579 node = self.deref(data, d)
580 kind = node.get("type")
581 if kind == "object":
582 result = self.buildObject(node, d)
583 elif kind == "array":
584 pass
585 _registry.register(d, self.version, result)
586 # XXX: This makes sure that the type gets added to the global
587 # _types dict even if no other type in the schema has a ref
588 # to it.
589 getRefType(d)
590
591 def buildObject(self, node, name=None, d=0):
592 # we don't need to build types recursively here
593 # they are all in definitions already
594 # we only want to include the type reference
595 # which we can derive from the name
596 struct = []
597 add = struct.append
598 props = node.get("properties")
599 pprops = node.get("patternProperties")
600 if props:
601 # Sort these so the __init__ arg list for each Type remains
602 # consistently ordered across regens of client.py
603 for p in sorted(props):
604 prop = props[p]
605 if "$ref" in prop:
606 add((p, refType(prop)))
607 else:
608 kind = prop['type']
609 if kind == "array":
610 add((p, self.buildArray(prop, d + 1)))
611 elif kind == "object":
612 struct.extend(self.buildObject(prop, p, d + 1))
613 else:
614 add((p, objType(prop)))
615 if pprops:
616 if ".*" not in pprops:
617 raise ValueError(
618 "Cannot handle actual pattern in patternProperties %s" %
619 pprops)
620 pprop = pprops[".*"]
621 if "$ref" in pprop:
622 add((name, Mapping[str, refType(pprop)]))
623 return struct
624 ppkind = pprop["type"]
625 if ppkind == "array":
626 add((name, self.buildArray(pprop, d + 1)))
627 else:
628 add((name, Mapping[str, SCHEMA_TO_PYTHON[ppkind]]))
629
630 if not struct and node.get('additionalProperties', False):
631 add((name, Mapping[str, SCHEMA_TO_PYTHON['object']]))
632
633 return struct
634
635 def buildArray(self, obj, d=0):
636 # return a sequence from an array in the schema
637 if "$ref" in obj:
638 return Sequence[refType(obj)]
639 else:
640 kind = obj.get("type")
641 if kind and kind == "array":
642 items = obj['items']
643 return self.buildArray(items, d + 1)
644 else:
645 return Sequence[objType(obj)]
646
647
648 def _getns():
649 ns = {'Type': Type,
650 'typing': typing,
651 'ReturnMapping': ReturnMapping
652 }
653 # Copy our types into the globals of the method
654 for facade in _registry:
655 ns[facade] = _registry.getObj(facade)
656 return ns
657
658
659 def make_factory(name):
660 if name in factories:
661 del factories[name]
662 factories[name].write("class {}(TypeFactory):\n pass\n\n".format(name))
663
664
665 def write_facades(captures, options):
666 """
667 Write the Facades to the appropriate _client<version>.py
668
669 """
670 for version in sorted(captures.keys()):
671 filename = "{}/_client{}.py".format(options.output_dir, version)
672 with open(filename, "w") as f:
673 f.write(HEADER)
674 f.write("from juju.client.facade import Type, ReturnMapping\n")
675 f.write("from juju.client._definitions import *\n\n")
676 for key in sorted(
677 [k for k in captures[version].keys() if "Facade" in k]):
678 print(captures[version][key], file=f)
679
680 # Return the last (most recent) version for use in other routines.
681 return version
682
683
684 def write_definitions(captures, options, version):
685 """
686 Write auxillary (non versioned) classes to
687 _definitions.py The auxillary classes currently get
688 written redudantly into each capture object, so we can look in
689 one of them -- we just use the last one from the loop above.
690
691 """
692 with open("{}/_definitions.py".format(options.output_dir), "w") as f:
693 f.write(HEADER)
694 f.write("from juju.client.facade import Type, ReturnMapping\n\n")
695 for key in sorted(
696 [k for k in captures[version].keys() if "Facade" not in k]):
697 print(captures[version][key], file=f)
698
699
700 def write_client(captures, options):
701 """
702 Write the TypeFactory classes to _client.py, along with some
703 imports and tables so that we can look up versioned Facades.
704
705 """
706 with open("{}/_client.py".format(options.output_dir), "w") as f:
707 f.write(HEADER)
708 f.write("from juju.client._definitions import *\n\n")
709 clients = ", ".join("_client{}".format(v) for v in captures)
710 f.write("from juju.client import " + clients + "\n\n")
711 f.write(CLIENT_TABLE.format(clients=",\n ".join(
712 ['"{}": _client{}'.format(v, v) for v in captures])))
713 f.write(LOOKUP_FACADE)
714 f.write(TYPE_FACTORY)
715 for key in sorted([k for k in factories.keys() if "Facade" in k]):
716 print(factories[key], file=f)
717
718
719 def write_version_map(options):
720 """
721 In order to work around
722 https://bugs.launchpad.net/juju/+bug/1682925, we build a map of
723 the facades that each version supports, and write it to disk here.
724
725 """
726 with open("{}/version_map.py".format(options.output_dir), "w") as f:
727 f.write(HEADER)
728 f.write("VERSION_MAP = {\n")
729 for juju_version in sorted(VERSION_MAP.keys()):
730 f.write(' "{}": {{\n'.format(juju_version))
731 for key in VERSION_MAP[juju_version]:
732 f.write(' "{}": {},\n'.format(
733 key, VERSION_MAP[juju_version][key]))
734 f.write(' },\n')
735 f.write("}\n")
736
737
738 def generate_facades(options):
739 captures = defaultdict(codegen.Capture)
740 schemas = {}
741 for p in sorted(glob(options.schema)):
742 if 'latest' in p:
743 juju_version = 'latest'
744 else:
745 try:
746 juju_version = re.search(JUJU_VERSION, p).group()
747 except AttributeError:
748 print("Cannot extract a juju version from {}".format(p))
749 print("Schemas must include a juju version in the filename")
750 raise SystemExit(1)
751
752 new_schemas = json.loads(Path(p).read_text("utf-8"))
753 schemas[juju_version] = [Schema(s) for s in new_schemas]
754
755 # Build all of the auxillary (unversioned) classes
756 # TODO: get rid of some of the excess trips through loops in the
757 # called functions.
758 for juju_version in sorted(schemas.keys()):
759 for schema in schemas[juju_version]:
760 schema.buildDefinitions()
761 buildTypes(schema, captures[schema.version])
762 VERSION_MAP[juju_version][schema.name] = schema.version
763
764 # Build the Facade classes
765 for juju_version in sorted(schemas.keys()):
766 for schema in schemas[juju_version]:
767 cls, source = buildFacade(schema)
768 cls_name = "{}Facade".format(schema.name)
769
770 captures[schema.version].clear(cls_name)
771 # Make the factory class for _client.py
772 make_factory(cls_name)
773 # Make the actual class
774 captures[schema.version][cls_name].write(source)
775 # Build the methods for each Facade class.
776 buildMethods(cls, captures[schema.version])
777 # Mark this Facade class as being done for this version --
778 # helps mitigate some excessive looping.
779 CLASSES[schema.name] = cls
780
781 return captures
782
783 def setup():
784 parser = argparse.ArgumentParser()
785 parser.add_argument("-s", "--schema", default="juju/client/schemas*")
786 parser.add_argument("-o", "--output_dir", default="juju/client")
787 options = parser.parse_args()
788 return options
789
790 def main():
791 options = setup()
792
793 # Generate some text blobs
794 captures = generate_facades(options)
795
796 # ... and write them out
797 last_version = write_facades(captures, options)
798 write_definitions(captures, options, last_version)
799 write_client(captures, options)
800 write_version_map(options)
801
802 if __name__ == '__main__':
803 main()