Merge changes Iacd2f028,I43a6d573,Ibb6c93bb
[osm/N2VC.git] / modules / libjuju / juju / client / facade.py
1 import argparse
2 import builtins
3 import functools
4 import json
5 import keyword
6 import pprint
7 import re
8 import textwrap
9 import typing
10 from collections import defaultdict
11 from glob import glob
12 from pathlib import Path
13 from typing import Any, Mapping, Sequence, TypeVar, Union
14
15 from . import codegen
16
17 _marker = object()
18
19 JUJU_VERSION = re.compile('[0-9]+\.[0-9-]+[\.\-][0-9a-z]+(\.[0-9]+)?')
20 # Workaround for https://bugs.launchpad.net/juju/+bug/1683906
21 NAUGHTY_CLASSES = ['ClientFacade', 'Client', 'FullStatus', 'ModelStatusInfo',
22 'ModelInfo']
23
24
25 # Map basic types to Python's typing with a callable
26 SCHEMA_TO_PYTHON = {
27 'string': str,
28 'integer': int,
29 'float': float,
30 'number': float,
31 'boolean': bool,
32 'object': Any,
33 }
34
35
36 # Friendly warning message to stick at the top of generated files.
37 HEADER = """\
38 # DO NOT CHANGE THIS FILE! This file is auto-generated by facade.py.
39 # Changes will be overwritten/lost when the file is regenerated.
40
41 """
42
43
44 # Classes and helper functions that we'll write to _client.py
45 LOOKUP_FACADE = '''
46 def lookup_facade(name, version):
47 """
48 Given a facade name and version, attempt to pull that facade out
49 of the correct client<version>.py file.
50
51 """
52 try:
53 facade = getattr(CLIENTS[str(version)], name)
54 except KeyError:
55 raise ImportError("No facades found for version {}".format(version))
56 except AttributeError:
57 raise ImportError(
58 "No facade with name '{}' in version {}".format(name, version))
59 return facade
60
61
62 '''
63
64 TYPE_FACTORY = '''
65 class TypeFactory:
66 @classmethod
67 def from_connection(cls, connection):
68 """
69 Given a connected Connection object, return an initialized and
70 connected instance of an API Interface matching the name of
71 this class.
72
73 @param connection: initialized Connection object.
74
75 """
76 version = connection.facades[cls.__name__[:-6]]
77
78 c = lookup_facade(cls.__name__, version)
79 c = c()
80 c.connect(connection)
81
82 return c
83
84
85 '''
86
87 CLIENT_TABLE = '''
88 CLIENTS = {{
89 {clients}
90 }}
91
92
93 '''
94
95
96 class KindRegistry(dict):
97 def register(self, name, version, obj):
98 self[name] = {version: {
99 "object": obj,
100 }}
101
102 def lookup(self, name, version=None):
103 """If version is omitted, max version is used"""
104 versions = self.get(name)
105 if not versions:
106 return None
107 if version:
108 return versions[version]
109 return versions[max(versions)]
110
111 def getObj(self, name, version=None):
112 result = self.lookup(name, version)
113 if result:
114 obj = result["object"]
115 return obj
116 return None
117
118
119 class TypeRegistry(dict):
120 def get(self, name):
121 # Two way mapping
122 refname = Schema.referenceName(name)
123 if refname not in self:
124 result = TypeVar(refname)
125 self[refname] = result
126 self[result] = refname
127
128 return self[refname]
129
130
131 _types = TypeRegistry()
132 _registry = KindRegistry()
133 CLASSES = {}
134 factories = codegen.Capture()
135
136
137 def booler(v):
138 if isinstance(v, str):
139 if v == "false":
140 return False
141 return bool(v)
142
143
144 def getRefType(ref):
145 return _types.get(ref)
146
147
148 def refType(obj):
149 return getRefType(obj["$ref"])
150
151
152 def objType(obj):
153 kind = obj.get('type')
154 if not kind:
155 raise ValueError("%s has no type" % obj)
156 result = SCHEMA_TO_PYTHON.get(kind)
157 if not result:
158 raise ValueError("%s has type %s" % (obj, kind))
159 return result
160
161
162 basic_types = [str, bool, int, float]
163
164
165 def name_to_py(name):
166 result = name.replace("-", "_")
167 result = result.lower()
168 if keyword.iskeyword(result) or result in dir(builtins):
169 result += "_"
170 return result
171
172
173 def strcast(kind, keep_builtins=False):
174 if (kind in basic_types or
175 type(kind) in basic_types) and keep_builtins is False:
176 return kind.__name__
177 if str(kind).startswith('~'):
178 return str(kind)[1:]
179 if issubclass(kind, typing.GenericMeta):
180 return str(kind)[1:]
181 return kind
182
183
184 class Args(list):
185 def __init__(self, defs):
186 self.defs = defs
187 if defs:
188 rtypes = _registry.getObj(_types[defs])
189 if len(rtypes) == 1:
190 if not self.do_explode(rtypes[0][1]):
191 for name, rtype in rtypes:
192 self.append((name, rtype))
193 else:
194 for name, rtype in rtypes:
195 self.append((name, rtype))
196
197 def do_explode(self, kind):
198 if kind in basic_types or type(kind) is typing.TypeVar:
199 return False
200 if not issubclass(kind, (typing.Sequence,
201 typing.Mapping)):
202 self.clear()
203 self.extend(Args(kind))
204 return True
205 return False
206
207 def PyToSchemaMapping(self):
208 m = {}
209 for n, rt in self:
210 m[name_to_py(n)] = n
211 return m
212
213 def SchemaToPyMapping(self):
214 m = {}
215 for n, tr in self:
216 m[n] = name_to_py(n)
217 return m
218
219 def _format(self, name, rtype, typed=True):
220 if typed:
221 return "{} : {}".format(
222 name_to_py(name),
223 strcast(rtype)
224 )
225 else:
226 return name_to_py(name)
227
228 def _get_arg_str(self, typed=False, joined=", "):
229 if self:
230 parts = []
231 for item in self:
232 parts.append(self._format(item[0], item[1], typed))
233 if joined:
234 return joined.join(parts)
235 return parts
236 return ''
237
238 def as_kwargs(self):
239 if self:
240 parts = []
241 for item in self:
242 parts.append('{}=None'.format(name_to_py(item[0])))
243 return ', '.join(parts)
244 return ''
245
246 def typed(self):
247 return self._get_arg_str(True)
248
249 def __str__(self):
250 return self._get_arg_str(False)
251
252 def get_doc(self):
253 return self._get_arg_str(True, "\n")
254
255
256 def buildTypes(schema, capture):
257 INDENT = " "
258 for kind in sorted((k for k in _types if not isinstance(k, str)),
259 key=lambda x: str(x)):
260 name = _types[kind]
261 if name in capture and name not in NAUGHTY_CLASSES:
262 continue
263 args = Args(kind)
264 # Write Factory class for _client.py
265 make_factory(name)
266 # Write actual class
267 source = ["""
268 class {}(Type):
269 _toSchema = {}
270 _toPy = {}
271 def __init__(self{}{}, **unknown_fields):
272 '''
273 {}
274 '''""".format(
275 name,
276 # pprint these to get stable ordering across regens
277 pprint.pformat(args.PyToSchemaMapping(), width=999),
278 pprint.pformat(args.SchemaToPyMapping(), width=999),
279 ", " if args else "",
280 args.as_kwargs(),
281 textwrap.indent(args.get_doc(), INDENT * 2))]
282
283 if not args:
284 source.append("{}pass".format(INDENT * 2))
285 else:
286 for arg in args:
287 arg_name = name_to_py(arg[0])
288 arg_type = arg[1]
289 arg_type_name = strcast(arg_type)
290 if arg_type in basic_types:
291 source.append("{}self.{} = {}".format(INDENT * 2,
292 arg_name,
293 arg_name))
294 elif type(arg_type) is typing.TypeVar:
295 source.append("{}self.{} = {}.from_json({}) "
296 "if {} else None".format(INDENT * 2,
297 arg_name,
298 arg_type_name,
299 arg_name,
300 arg_name))
301 elif issubclass(arg_type, typing.Sequence):
302 value_type = (
303 arg_type_name.__parameters__[0]
304 if len(arg_type_name.__parameters__)
305 else None
306 )
307 if type(value_type) is typing.TypeVar:
308 source.append(
309 "{}self.{} = [{}.from_json(o) "
310 "for o in {} or []]".format(INDENT * 2,
311 arg_name,
312 strcast(value_type),
313 arg_name))
314 else:
315 source.append("{}self.{} = {}".format(INDENT * 2,
316 arg_name,
317 arg_name))
318 elif issubclass(arg_type, typing.Mapping):
319 value_type = (
320 arg_type_name.__parameters__[1]
321 if len(arg_type_name.__parameters__) > 1
322 else None
323 )
324 if type(value_type) is typing.TypeVar:
325 source.append(
326 "{}self.{} = {{k: {}.from_json(v) "
327 "for k, v in ({} or dict()).items()}}".format(
328 INDENT * 2,
329 arg_name,
330 strcast(value_type),
331 arg_name))
332 else:
333 source.append("{}self.{} = {}".format(INDENT * 2,
334 arg_name,
335 arg_name))
336 else:
337 source.append("{}self.{} = {}".format(INDENT * 2,
338 arg_name,
339 arg_name))
340
341 source = "\n".join(source)
342 capture.clear(name)
343 capture[name].write(source)
344 capture[name].write("\n\n")
345 co = compile(source, __name__, "exec")
346 ns = _getns()
347 exec(co, ns)
348 cls = ns[name]
349 CLASSES[name] = cls
350
351
352 def retspec(defs):
353 # return specs
354 # only return 1, so if there is more than one type
355 # we need to include a union
356 # In truth there is only 1 return
357 # Error or the expected Type
358 if not defs:
359 return None
360 if defs in basic_types:
361 return strcast(defs, False)
362 rtypes = _registry.getObj(_types[defs])
363 if not rtypes:
364 return None
365 if len(rtypes) > 1:
366 return Union[tuple([strcast(r[1], True) for r in rtypes])]
367 return strcast(rtypes[0][1], False)
368
369
370 def return_type(defs):
371 if not defs:
372 return None
373 rtypes = _registry.getObj(_types[defs])
374 if not rtypes:
375 return None
376 if len(rtypes) > 1:
377 for n, t in rtypes:
378 if n == "Error":
379 continue
380 return t
381 return rtypes[0][1]
382
383
384 def type_anno_func(func, defs, is_result=False):
385 annos = {}
386 if not defs:
387 return func
388 rtypes = _registry.getObj(_types[defs])
389 if is_result:
390 kn = "return"
391 if not rtypes:
392 annos[kn] = None
393 elif len(rtypes) > 1:
394 annos[kn] = Union[tuple([r[1] for r in rtypes])]
395 else:
396 annos[kn] = rtypes[0][1]
397 else:
398 for name, rtype in rtypes:
399 name = name_to_py(name)
400 annos[name] = rtype
401 func.__annotations__.update(annos)
402 return func
403
404
405 def ReturnMapping(cls):
406 # Annotate the method with a return Type
407 # so the value can be cast
408 def decorator(f):
409 @functools.wraps(f)
410 async def wrapper(*args, **kwargs):
411 nonlocal cls
412 reply = await f(*args, **kwargs)
413 if cls is None:
414 return reply
415 if 'error' in reply:
416 cls = CLASSES['Error']
417 if issubclass(cls, typing.Sequence):
418 result = []
419 item_cls = cls.__parameters__[0]
420 for item in reply:
421 result.append(item_cls.from_json(item))
422 """
423 if 'error' in item:
424 cls = CLASSES['Error']
425 else:
426 cls = item_cls
427 result.append(cls.from_json(item))
428 """
429 else:
430 result = cls.from_json(reply['response'])
431
432 return result
433 return wrapper
434 return decorator
435
436
437 def makeFunc(cls, name, params, result, _async=True):
438 INDENT = " "
439 args = Args(params)
440 assignments = []
441 toschema = args.PyToSchemaMapping()
442 for arg in args._get_arg_str(False, False):
443 assignments.append("{}_params[\'{}\'] = {}".format(INDENT,
444 toschema[arg],
445 arg))
446 assignments = "\n".join(assignments)
447 res = retspec(result)
448 source = """
449
450 @ReturnMapping({rettype})
451 {_async}def {name}(self{argsep}{args}):
452 '''
453 {docstring}
454 Returns -> {res}
455 '''
456 # map input types to rpc msg
457 _params = dict()
458 msg = dict(type='{cls.name}',
459 request='{name}',
460 version={cls.version},
461 params=_params)
462 {assignments}
463 reply = {_await}self.rpc(msg)
464 return reply
465
466 """
467
468 fsource = source.format(_async="async " if _async else "",
469 name=name,
470 argsep=", " if args else "",
471 args=args,
472 res=res,
473 rettype=result.__name__ if result else None,
474 docstring=textwrap.indent(args.get_doc(), INDENT),
475 cls=cls,
476 assignments=assignments,
477 _await="await " if _async else "")
478 ns = _getns()
479 exec(fsource, ns)
480 func = ns[name]
481 return func, fsource
482
483
484 def buildMethods(cls, capture):
485 properties = cls.schema['properties']
486 for methodname in sorted(properties):
487 method, source = _buildMethod(cls, methodname)
488 setattr(cls, methodname, method)
489 capture["{}Facade".format(cls.__name__)].write(source, depth=1)
490
491
492 def _buildMethod(cls, name):
493 params = None
494 result = None
495 method = cls.schema['properties'][name]
496 if 'properties' in method:
497 prop = method['properties']
498 spec = prop.get('Params')
499 if spec:
500 params = _types.get(spec['$ref'])
501 spec = prop.get('Result')
502 if spec:
503 if '$ref' in spec:
504 result = _types.get(spec['$ref'])
505 else:
506 result = SCHEMA_TO_PYTHON[spec['type']]
507 return makeFunc(cls, name, params, result)
508
509
510 def buildFacade(schema):
511 cls = type(schema.name, (Type,), dict(name=schema.name,
512 version=schema.version,
513 schema=schema))
514 source = """
515 class {name}Facade(Type):
516 name = '{name}'
517 version = {version}
518 schema = {schema}
519 """.format(name=schema.name,
520 version=schema.version,
521 schema=textwrap.indent(pprint.pformat(schema), " "))
522 return cls, source
523
524
525 class TypeEncoder(json.JSONEncoder):
526 def default(self, obj):
527 if isinstance(obj, Type):
528 return obj.serialize()
529 return json.JSONEncoder.default(self, obj)
530
531
532 class Type:
533 def connect(self, connection):
534 self.connection = connection
535
536 async def rpc(self, msg):
537 result = await self.connection.rpc(msg, encoder=TypeEncoder)
538 return result
539
540 @classmethod
541 def from_json(cls, data):
542 if isinstance(data, cls):
543 return data
544 if isinstance(data, str):
545 try:
546 data = json.loads(data)
547 except json.JSONDecodeError:
548 raise
549 d = {}
550 for k, v in (data or {}).items():
551 d[cls._toPy.get(k, k)] = v
552
553 try:
554 return cls(**d)
555 except TypeError:
556 raise
557
558 def serialize(self):
559 d = {}
560 for attr, tgt in self._toSchema.items():
561 d[tgt] = getattr(self, attr)
562 return d
563
564 def to_json(self):
565 return json.dumps(self.serialize(), cls=TypeEncoder, sort_keys=True)
566
567
568 class Schema(dict):
569 def __init__(self, schema):
570 self.name = schema['Name']
571 self.version = schema['Version']
572 self.update(schema['Schema'])
573
574 @classmethod
575 def referenceName(cls, ref):
576 if ref.startswith("#/definitions/"):
577 ref = ref.rsplit("/", 1)[-1]
578 return ref
579
580 def resolveDefinition(self, ref):
581 return self['definitions'][self.referenceName(ref)]
582
583 def deref(self, prop, name):
584 if not isinstance(prop, dict):
585 raise TypeError(prop)
586 if "$ref" not in prop:
587 return prop
588
589 target = self.resolveDefinition(prop["$ref"])
590 return target
591
592 def buildDefinitions(self):
593 # here we are building the types out
594 # anything in definitions is a type
595 # but these may contain references themselves
596 # so we dfs to the bottom and build upwards
597 # when a types is already in the registry
598 defs = self.get('definitions')
599 if not defs:
600 return
601 for d, data in defs.items():
602 if d in _registry and d not in NAUGHTY_CLASSES:
603 continue
604 node = self.deref(data, d)
605 kind = node.get("type")
606 if kind == "object":
607 result = self.buildObject(node, d)
608 elif kind == "array":
609 pass
610 _registry.register(d, self.version, result)
611 # XXX: This makes sure that the type gets added to the global
612 # _types dict even if no other type in the schema has a ref
613 # to it.
614 getRefType(d)
615
616 def buildObject(self, node, name=None, d=0):
617 # we don't need to build types recursively here
618 # they are all in definitions already
619 # we only want to include the type reference
620 # which we can derive from the name
621 struct = []
622 add = struct.append
623 props = node.get("properties")
624 pprops = node.get("patternProperties")
625 if props:
626 # Sort these so the __init__ arg list for each Type remains
627 # consistently ordered across regens of client.py
628 for p in sorted(props):
629 prop = props[p]
630 if "$ref" in prop:
631 add((p, refType(prop)))
632 else:
633 kind = prop['type']
634 if kind == "array":
635 add((p, self.buildArray(prop, d + 1)))
636 elif kind == "object":
637 struct.extend(self.buildObject(prop, p, d + 1))
638 else:
639 add((p, objType(prop)))
640 if pprops:
641 if ".*" not in pprops:
642 raise ValueError(
643 "Cannot handle actual pattern in patternProperties %s" %
644 pprops)
645 pprop = pprops[".*"]
646 if "$ref" in pprop:
647 add((name, Mapping[str, refType(pprop)]))
648 return struct
649 ppkind = pprop["type"]
650 if ppkind == "array":
651 add((name, self.buildArray(pprop, d + 1)))
652 else:
653 add((name, Mapping[str, SCHEMA_TO_PYTHON[ppkind]]))
654
655 if not struct and node.get('additionalProperties', False):
656 add((name, Mapping[str, SCHEMA_TO_PYTHON['object']]))
657
658 return struct
659
660 def buildArray(self, obj, d=0):
661 # return a sequence from an array in the schema
662 if "$ref" in obj:
663 return Sequence[refType(obj)]
664 else:
665 kind = obj.get("type")
666 if kind and kind == "array":
667 items = obj['items']
668 return self.buildArray(items, d + 1)
669 else:
670 return Sequence[objType(obj)]
671
672
673 def _getns():
674 ns = {'Type': Type,
675 'typing': typing,
676 'ReturnMapping': ReturnMapping
677 }
678 # Copy our types into the globals of the method
679 for facade in _registry:
680 ns[facade] = _registry.getObj(facade)
681 return ns
682
683
684 def make_factory(name):
685 if name in factories:
686 del factories[name]
687 factories[name].write("class {}(TypeFactory):\n pass\n\n".format(name))
688
689
690 def write_facades(captures, options):
691 """
692 Write the Facades to the appropriate _client<version>.py
693
694 """
695 for version in sorted(captures.keys()):
696 filename = "{}/_client{}.py".format(options.output_dir, version)
697 with open(filename, "w") as f:
698 f.write(HEADER)
699 f.write("from juju.client.facade import Type, ReturnMapping\n")
700 f.write("from juju.client._definitions import *\n\n")
701 for key in sorted(
702 [k for k in captures[version].keys() if "Facade" in k]):
703 print(captures[version][key], file=f)
704
705 # Return the last (most recent) version for use in other routines.
706 return version
707
708
709 def write_definitions(captures, options, version):
710 """
711 Write auxillary (non versioned) classes to
712 _definitions.py The auxillary classes currently get
713 written redudantly into each capture object, so we can look in
714 one of them -- we just use the last one from the loop above.
715
716 """
717 with open("{}/_definitions.py".format(options.output_dir), "w") as f:
718 f.write(HEADER)
719 f.write("from juju.client.facade import Type, ReturnMapping\n\n")
720 for key in sorted(
721 [k for k in captures[version].keys() if "Facade" not in k]):
722 print(captures[version][key], file=f)
723
724
725 def write_client(captures, options):
726 """
727 Write the TypeFactory classes to _client.py, along with some
728 imports and tables so that we can look up versioned Facades.
729
730 """
731 with open("{}/_client.py".format(options.output_dir), "w") as f:
732 f.write(HEADER)
733 f.write("from juju.client._definitions import *\n\n")
734 clients = ", ".join("_client{}".format(v) for v in captures)
735 f.write("from juju.client import " + clients + "\n\n")
736 f.write(CLIENT_TABLE.format(clients=",\n ".join(
737 ['"{}": _client{}'.format(v, v) for v in captures])))
738 f.write(LOOKUP_FACADE)
739 f.write(TYPE_FACTORY)
740 for key in sorted([k for k in factories.keys() if "Facade" in k]):
741 print(factories[key], file=f)
742
743
744 def generate_facades(options):
745 captures = defaultdict(codegen.Capture)
746 schemas = {}
747 for p in sorted(glob(options.schema)):
748 if 'latest' in p:
749 juju_version = 'latest'
750 else:
751 try:
752 juju_version = re.search(JUJU_VERSION, p).group()
753 except AttributeError:
754 print("Cannot extract a juju version from {}".format(p))
755 print("Schemas must include a juju version in the filename")
756 raise SystemExit(1)
757
758 new_schemas = json.loads(Path(p).read_text("utf-8"))
759 schemas[juju_version] = [Schema(s) for s in new_schemas]
760
761 # Build all of the auxillary (unversioned) classes
762 # TODO: get rid of some of the excess trips through loops in the
763 # called functions.
764 for juju_version in sorted(schemas.keys()):
765 for schema in schemas[juju_version]:
766 schema.buildDefinitions()
767 buildTypes(schema, captures[schema.version])
768
769 # Build the Facade classes
770 for juju_version in sorted(schemas.keys()):
771 for schema in schemas[juju_version]:
772 cls, source = buildFacade(schema)
773 cls_name = "{}Facade".format(schema.name)
774
775 captures[schema.version].clear(cls_name)
776 # Make the factory class for _client.py
777 make_factory(cls_name)
778 # Make the actual class
779 captures[schema.version][cls_name].write(source)
780 # Build the methods for each Facade class.
781 buildMethods(cls, captures[schema.version])
782 # Mark this Facade class as being done for this version --
783 # helps mitigate some excessive looping.
784 CLASSES[schema.name] = cls
785
786 return captures
787
788
789 def setup():
790 parser = argparse.ArgumentParser()
791 parser.add_argument("-s", "--schema", default="juju/client/schemas*")
792 parser.add_argument("-o", "--output_dir", default="juju/client")
793 options = parser.parse_args()
794 return options
795
796
797 def main():
798 options = setup()
799
800 # Generate some text blobs
801 captures = generate_facades(options)
802
803 # ... and write them out
804 last_version = write_facades(captures, options)
805 write_definitions(captures, options, last_version)
806 write_client(captures, options)
807
808
809 if __name__ == '__main__':
810 main()