K8s Juju connector
[osm/N2VC.git] / modules / libjuju / juju / client / facade.py
1 import argparse
2 import builtins
3 import functools
4 import json
5 import keyword
6 import pprint
7 import re
8 import textwrap
9 import typing
10 from collections import defaultdict
11 from glob import glob
12 from pathlib import Path
13 from typing import Any, Mapping, Sequence, TypeVar, Union
14
15 from . import codegen
16
17 _marker = object()
18
19 JUJU_VERSION = re.compile(r'[0-9]+\.[0-9-]+[\.\-][0-9a-z]+(\.[0-9]+)?')
20 # Workaround for https://bugs.launchpad.net/juju/+bug/1683906
21 NAUGHTY_CLASSES = ['ClientFacade', 'Client', 'FullStatus', 'ModelStatusInfo',
22 'ModelInfo', 'ApplicationDeploy']
23
24
25 # Map basic types to Python's typing with a callable
26 SCHEMA_TO_PYTHON = {
27 'string': str,
28 'integer': int,
29 'float': float,
30 'number': float,
31 'boolean': bool,
32 'object': Any,
33 }
34
35
36 # Friendly warning message to stick at the top of generated files.
37 HEADER = """\
38 # DO NOT CHANGE THIS FILE! This file is auto-generated by facade.py.
39 # Changes will be overwritten/lost when the file is regenerated.
40
41 """
42
43
44 # Classes and helper functions that we'll write to _client.py
45 LOOKUP_FACADE = '''
46 def lookup_facade(name, version):
47 """
48 Given a facade name and version, attempt to pull that facade out
49 of the correct client<version>.py file.
50
51 """
52 for _version in range(int(version), 0, -1):
53 try:
54 facade = getattr(CLIENTS[str(_version)], name)
55 return facade
56 except (KeyError, AttributeError):
57 continue
58 else:
59 raise ImportError("No supported version for facade: "
60 "{}".format(name))
61
62
63 '''
64
65 TYPE_FACTORY = '''
66 class TypeFactory:
67 @classmethod
68 def from_connection(cls, connection):
69 """
70 Given a connected Connection object, return an initialized and
71 connected instance of an API Interface matching the name of
72 this class.
73
74 @param connection: initialized Connection object.
75
76 """
77 facade_name = cls.__name__
78 if not facade_name.endswith('Facade'):
79 raise TypeError('Unexpected class name: {}'.format(facade_name))
80 facade_name = facade_name[:-len('Facade')]
81 version = connection.facades.get(facade_name)
82 if version is None:
83 raise Exception('No facade {} in facades {}'.format(facade_name,
84 connection.facades))
85
86 c = lookup_facade(cls.__name__, version)
87 c = c()
88 c.connect(connection)
89
90 return c
91
92
93 '''
94
95 CLIENT_TABLE = '''
96 CLIENTS = {{
97 {clients}
98 }}
99
100
101 '''
102
103
104 class KindRegistry(dict):
105 def register(self, name, version, obj):
106 self[name] = {version: {
107 "object": obj,
108 }}
109
110 def lookup(self, name, version=None):
111 """If version is omitted, max version is used"""
112 versions = self.get(name)
113 if not versions:
114 return None
115 if version:
116 return versions[version]
117 return versions[max(versions)]
118
119 def getObj(self, name, version=None):
120 result = self.lookup(name, version)
121 if result:
122 obj = result["object"]
123 return obj
124 return None
125
126
127 class TypeRegistry(dict):
128 def get(self, name):
129 # Two way mapping
130 refname = Schema.referenceName(name)
131 if refname not in self:
132 result = TypeVar(refname)
133 self[refname] = result
134 self[result] = refname
135
136 return self[refname]
137
138
139 _types = TypeRegistry()
140 _registry = KindRegistry()
141 CLASSES = {}
142 factories = codegen.Capture()
143
144
145 def booler(v):
146 if isinstance(v, str):
147 if v == "false":
148 return False
149 return bool(v)
150
151
152 def getRefType(ref):
153 return _types.get(ref)
154
155
156 def refType(obj):
157 return getRefType(obj["$ref"])
158
159
160 def objType(obj):
161 kind = obj.get('type')
162 if not kind:
163 raise ValueError("%s has no type" % obj)
164 result = SCHEMA_TO_PYTHON.get(kind)
165 if not result:
166 raise ValueError("%s has type %s" % (obj, kind))
167 return result
168
169
170 basic_types = [str, bool, int, float]
171
172
173 def name_to_py(name):
174 result = name.replace("-", "_")
175 result = result.lower()
176 if keyword.iskeyword(result) or result in dir(builtins):
177 result += "_"
178 return result
179
180
181 def strcast(kind, keep_builtins=False):
182 if (kind in basic_types or
183 type(kind) in basic_types) and keep_builtins is False:
184 return kind.__name__
185 if str(kind).startswith('~'):
186 return str(kind)[1:]
187 if issubclass(kind, typing.GenericMeta):
188 return str(kind)[1:]
189 return kind
190
191
192 class Args(list):
193 def __init__(self, defs):
194 self.defs = defs
195 if defs:
196 rtypes = _registry.getObj(_types[defs])
197 if len(rtypes) == 1:
198 if not self.do_explode(rtypes[0][1]):
199 for name, rtype in rtypes:
200 self.append((name, rtype))
201 else:
202 for name, rtype in rtypes:
203 self.append((name, rtype))
204
205 def do_explode(self, kind):
206 if kind in basic_types or type(kind) is typing.TypeVar:
207 return False
208 if not issubclass(kind, (typing.Sequence,
209 typing.Mapping)):
210 self.clear()
211 self.extend(Args(kind))
212 return True
213 return False
214
215 def PyToSchemaMapping(self):
216 m = {}
217 for n, rt in self:
218 m[name_to_py(n)] = n
219 return m
220
221 def SchemaToPyMapping(self):
222 m = {}
223 for n, tr in self:
224 m[n] = name_to_py(n)
225 return m
226
227 def _format(self, name, rtype, typed=True):
228 if typed:
229 return "{} : {}".format(
230 name_to_py(name),
231 strcast(rtype)
232 )
233 else:
234 return name_to_py(name)
235
236 def _get_arg_str(self, typed=False, joined=", "):
237 if self:
238 parts = []
239 for item in self:
240 parts.append(self._format(item[0], item[1], typed))
241 if joined:
242 return joined.join(parts)
243 return parts
244 return ''
245
246 def as_kwargs(self):
247 if self:
248 parts = []
249 for item in self:
250 parts.append('{}=None'.format(name_to_py(item[0])))
251 return ', '.join(parts)
252 return ''
253
254 def typed(self):
255 return self._get_arg_str(True)
256
257 def __str__(self):
258 return self._get_arg_str(False)
259
260 def get_doc(self):
261 return self._get_arg_str(True, "\n")
262
263
264 def buildTypes(schema, capture):
265 INDENT = " "
266 for kind in sorted((k for k in _types if not isinstance(k, str)),
267 key=lambda x: str(x)):
268 name = _types[kind]
269 if name in capture and name not in NAUGHTY_CLASSES:
270 continue
271 args = Args(kind)
272 # Write Factory class for _client.py
273 make_factory(name)
274 # Write actual class
275 source = ["""
276 class {}(Type):
277 _toSchema = {}
278 _toPy = {}
279 def __init__(self{}{}, **unknown_fields):
280 '''
281 {}
282 '''""".format(
283 name,
284 # pprint these to get stable ordering across regens
285 pprint.pformat(args.PyToSchemaMapping(), width=999),
286 pprint.pformat(args.SchemaToPyMapping(), width=999),
287 ", " if args else "",
288 args.as_kwargs(),
289 textwrap.indent(args.get_doc(), INDENT * 2))]
290
291 if not args:
292 source.append("{}pass".format(INDENT * 2))
293 else:
294 for arg in args:
295 arg_name = name_to_py(arg[0])
296 arg_type = arg[1]
297 arg_type_name = strcast(arg_type)
298 if arg_type in basic_types:
299 source.append("{}self.{} = {}".format(INDENT * 2,
300 arg_name,
301 arg_name))
302 elif type(arg_type) is typing.TypeVar:
303 source.append("{}self.{} = {}.from_json({}) "
304 "if {} else None".format(INDENT * 2,
305 arg_name,
306 arg_type_name,
307 arg_name,
308 arg_name))
309 elif issubclass(arg_type, typing.Sequence):
310 value_type = (
311 arg_type_name.__parameters__[0]
312 if len(arg_type_name.__parameters__)
313 else None
314 )
315 if type(value_type) is typing.TypeVar:
316 source.append(
317 "{}self.{} = [{}.from_json(o) "
318 "for o in {} or []]".format(INDENT * 2,
319 arg_name,
320 strcast(value_type),
321 arg_name))
322 else:
323 source.append("{}self.{} = {}".format(INDENT * 2,
324 arg_name,
325 arg_name))
326 elif issubclass(arg_type, typing.Mapping):
327 value_type = (
328 arg_type_name.__parameters__[1]
329 if len(arg_type_name.__parameters__) > 1
330 else None
331 )
332 if type(value_type) is typing.TypeVar:
333 source.append(
334 "{}self.{} = {{k: {}.from_json(v) "
335 "for k, v in ({} or dict()).items()}}".format(
336 INDENT * 2,
337 arg_name,
338 strcast(value_type),
339 arg_name))
340 else:
341 source.append("{}self.{} = {}".format(INDENT * 2,
342 arg_name,
343 arg_name))
344 else:
345 source.append("{}self.{} = {}".format(INDENT * 2,
346 arg_name,
347 arg_name))
348
349 source = "\n".join(source)
350 capture.clear(name)
351 capture[name].write(source)
352 capture[name].write("\n\n")
353 co = compile(source, __name__, "exec")
354 ns = _getns()
355 exec(co, ns)
356 cls = ns[name]
357 CLASSES[name] = cls
358
359
360 def retspec(defs):
361 # return specs
362 # only return 1, so if there is more than one type
363 # we need to include a union
364 # In truth there is only 1 return
365 # Error or the expected Type
366 if not defs:
367 return None
368 if defs in basic_types:
369 return strcast(defs, False)
370 rtypes = _registry.getObj(_types[defs])
371 if not rtypes:
372 return None
373 if len(rtypes) > 1:
374 return Union[tuple([strcast(r[1], True) for r in rtypes])]
375 return strcast(rtypes[0][1], False)
376
377
378 def return_type(defs):
379 if not defs:
380 return None
381 rtypes = _registry.getObj(_types[defs])
382 if not rtypes:
383 return None
384 if len(rtypes) > 1:
385 for n, t in rtypes:
386 if n == "Error":
387 continue
388 return t
389 return rtypes[0][1]
390
391
392 def type_anno_func(func, defs, is_result=False):
393 annos = {}
394 if not defs:
395 return func
396 rtypes = _registry.getObj(_types[defs])
397 if is_result:
398 kn = "return"
399 if not rtypes:
400 annos[kn] = None
401 elif len(rtypes) > 1:
402 annos[kn] = Union[tuple([r[1] for r in rtypes])]
403 else:
404 annos[kn] = rtypes[0][1]
405 else:
406 for name, rtype in rtypes:
407 name = name_to_py(name)
408 annos[name] = rtype
409 func.__annotations__.update(annos)
410 return func
411
412
413 def ReturnMapping(cls):
414 # Annotate the method with a return Type
415 # so the value can be cast
416 def decorator(f):
417 @functools.wraps(f)
418 async def wrapper(*args, **kwargs):
419 nonlocal cls
420 reply = await f(*args, **kwargs)
421 if cls is None:
422 return reply
423 if 'error' in reply:
424 cls = CLASSES['Error']
425 if issubclass(cls, typing.Sequence):
426 result = []
427 item_cls = cls.__parameters__[0]
428 for item in reply:
429 result.append(item_cls.from_json(item))
430 """
431 if 'error' in item:
432 cls = CLASSES['Error']
433 else:
434 cls = item_cls
435 result.append(cls.from_json(item))
436 """
437 else:
438 result = cls.from_json(reply['response'])
439
440 return result
441 return wrapper
442 return decorator
443
444
445 def makeFunc(cls, name, params, result, _async=True):
446 INDENT = " "
447 args = Args(params)
448 assignments = []
449 toschema = args.PyToSchemaMapping()
450 for arg in args._get_arg_str(False, False):
451 assignments.append("{}_params[\'{}\'] = {}".format(INDENT,
452 toschema[arg],
453 arg))
454 assignments = "\n".join(assignments)
455 res = retspec(result)
456 source = """
457
458 @ReturnMapping({rettype})
459 {_async}def {name}(self{argsep}{args}):
460 '''
461 {docstring}
462 Returns -> {res}
463 '''
464 # map input types to rpc msg
465 _params = dict()
466 msg = dict(type='{cls.name}',
467 request='{name}',
468 version={cls.version},
469 params=_params)
470 {assignments}
471 reply = {_await}self.rpc(msg)
472 return reply
473
474 """
475
476 fsource = source.format(_async="async " if _async else "",
477 name=name,
478 argsep=", " if args else "",
479 args=args,
480 res=res,
481 rettype=result.__name__ if result else None,
482 docstring=textwrap.indent(args.get_doc(), INDENT),
483 cls=cls,
484 assignments=assignments,
485 _await="await " if _async else "")
486 ns = _getns()
487 exec(fsource, ns)
488 func = ns[name]
489 return func, fsource
490
491
492 def buildMethods(cls, capture):
493 properties = cls.schema['properties']
494 for methodname in sorted(properties):
495 method, source = _buildMethod(cls, methodname)
496 setattr(cls, methodname, method)
497 capture["{}Facade".format(cls.__name__)].write(source, depth=1)
498
499
500 def _buildMethod(cls, name):
501 params = None
502 result = None
503 method = cls.schema['properties'][name]
504 if 'properties' in method:
505 prop = method['properties']
506 spec = prop.get('Params')
507 if spec:
508 params = _types.get(spec['$ref'])
509 spec = prop.get('Result')
510 if spec:
511 if '$ref' in spec:
512 result = _types.get(spec['$ref'])
513 else:
514 result = SCHEMA_TO_PYTHON[spec['type']]
515 return makeFunc(cls, name, params, result)
516
517
518 def buildFacade(schema):
519 cls = type(schema.name, (Type,), dict(name=schema.name,
520 version=schema.version,
521 schema=schema))
522 source = """
523 class {name}Facade(Type):
524 name = '{name}'
525 version = {version}
526 schema = {schema}
527 """.format(name=schema.name,
528 version=schema.version,
529 schema=textwrap.indent(pprint.pformat(schema), " "))
530 return cls, source
531
532
533 class TypeEncoder(json.JSONEncoder):
534 def default(self, obj):
535 if isinstance(obj, Type):
536 return obj.serialize()
537 return json.JSONEncoder.default(self, obj)
538
539
540 class Type:
541 def connect(self, connection):
542 self.connection = connection
543
544 async def rpc(self, msg):
545 result = await self.connection.rpc(msg, encoder=TypeEncoder)
546 return result
547
548 @classmethod
549 def from_json(cls, data):
550 if isinstance(data, cls):
551 return data
552 if isinstance(data, str):
553 try:
554 data = json.loads(data)
555 except json.JSONDecodeError:
556 raise
557 d = {}
558 for k, v in (data or {}).items():
559 d[cls._toPy.get(k, k)] = v
560
561 try:
562 return cls(**d)
563 except TypeError:
564 raise
565
566 def serialize(self):
567 d = {}
568 for attr, tgt in self._toSchema.items():
569 d[tgt] = getattr(self, attr)
570 return d
571
572 def to_json(self):
573 return json.dumps(self.serialize(), cls=TypeEncoder, sort_keys=True)
574
575
576 class Schema(dict):
577 def __init__(self, schema):
578 self.name = schema['Name']
579 self.version = schema['Version']
580 self.update(schema['Schema'])
581
582 @classmethod
583 def referenceName(cls, ref):
584 if ref.startswith("#/definitions/"):
585 ref = ref.rsplit("/", 1)[-1]
586 return ref
587
588 def resolveDefinition(self, ref):
589 return self['definitions'][self.referenceName(ref)]
590
591 def deref(self, prop, name):
592 if not isinstance(prop, dict):
593 raise TypeError(prop)
594 if "$ref" not in prop:
595 return prop
596
597 target = self.resolveDefinition(prop["$ref"])
598 return target
599
600 def buildDefinitions(self):
601 # here we are building the types out
602 # anything in definitions is a type
603 # but these may contain references themselves
604 # so we dfs to the bottom and build upwards
605 # when a types is already in the registry
606 defs = self.get('definitions')
607 if not defs:
608 return
609 for d, data in defs.items():
610 if d in _registry and d not in NAUGHTY_CLASSES:
611 continue
612 node = self.deref(data, d)
613 kind = node.get("type")
614 if kind == "object":
615 result = self.buildObject(node, d)
616 elif kind == "array":
617 pass
618 _registry.register(d, self.version, result)
619 # XXX: This makes sure that the type gets added to the global
620 # _types dict even if no other type in the schema has a ref
621 # to it.
622 getRefType(d)
623
624 def buildObject(self, node, name=None, d=0):
625 # we don't need to build types recursively here
626 # they are all in definitions already
627 # we only want to include the type reference
628 # which we can derive from the name
629 struct = []
630 add = struct.append
631 props = node.get("properties")
632 pprops = node.get("patternProperties")
633 if props:
634 # Sort these so the __init__ arg list for each Type remains
635 # consistently ordered across regens of client.py
636 for p in sorted(props):
637 prop = props[p]
638 if "$ref" in prop:
639 add((p, refType(prop)))
640 else:
641 kind = prop['type']
642 if kind == "array":
643 add((p, self.buildArray(prop, d + 1)))
644 elif kind == "object":
645 struct.extend(self.buildObject(prop, p, d + 1))
646 else:
647 add((p, objType(prop)))
648 if pprops:
649 if ".*" not in pprops:
650 raise ValueError(
651 "Cannot handle actual pattern in patternProperties %s" %
652 pprops)
653 pprop = pprops[".*"]
654 if "$ref" in pprop:
655 add((name, Mapping[str, refType(pprop)]))
656 return struct
657 ppkind = pprop["type"]
658 if ppkind == "array":
659 add((name, self.buildArray(pprop, d + 1)))
660 else:
661 add((name, Mapping[str, SCHEMA_TO_PYTHON[ppkind]]))
662
663 if not struct and node.get('additionalProperties', False):
664 add((name, Mapping[str, SCHEMA_TO_PYTHON['object']]))
665
666 return struct
667
668 def buildArray(self, obj, d=0):
669 # return a sequence from an array in the schema
670 if "$ref" in obj:
671 return Sequence[refType(obj)]
672 else:
673 kind = obj.get("type")
674 if kind and kind == "array":
675 items = obj['items']
676 return self.buildArray(items, d + 1)
677 else:
678 return Sequence[objType(obj)]
679
680
681 def _getns():
682 ns = {'Type': Type,
683 'typing': typing,
684 'ReturnMapping': ReturnMapping
685 }
686 # Copy our types into the globals of the method
687 for facade in _registry:
688 ns[facade] = _registry.getObj(facade)
689 return ns
690
691
692 def make_factory(name):
693 if name in factories:
694 del factories[name]
695 factories[name].write("class {}(TypeFactory):\n pass\n\n".format(name))
696
697
698 def write_facades(captures, options):
699 """
700 Write the Facades to the appropriate _client<version>.py
701
702 """
703 for version in sorted(captures.keys()):
704 filename = "{}/_client{}.py".format(options.output_dir, version)
705 with open(filename, "w") as f:
706 f.write(HEADER)
707 f.write("from juju.client.facade import Type, ReturnMapping\n")
708 f.write("from juju.client._definitions import *\n\n")
709 for key in sorted(
710 [k for k in captures[version].keys() if "Facade" in k]):
711 print(captures[version][key], file=f)
712
713 # Return the last (most recent) version for use in other routines.
714 return version
715
716
717 def write_definitions(captures, options, version):
718 """
719 Write auxillary (non versioned) classes to
720 _definitions.py The auxillary classes currently get
721 written redudantly into each capture object, so we can look in
722 one of them -- we just use the last one from the loop above.
723
724 """
725 with open("{}/_definitions.py".format(options.output_dir), "w") as f:
726 f.write(HEADER)
727 f.write("from juju.client.facade import Type, ReturnMapping\n\n")
728 for key in sorted(
729 [k for k in captures[version].keys() if "Facade" not in k]):
730 print(captures[version][key], file=f)
731
732
733 def write_client(captures, options):
734 """
735 Write the TypeFactory classes to _client.py, along with some
736 imports and tables so that we can look up versioned Facades.
737
738 """
739 with open("{}/_client.py".format(options.output_dir), "w") as f:
740 f.write(HEADER)
741 f.write("from juju.client._definitions import *\n\n")
742 clients = ", ".join("_client{}".format(v) for v in captures)
743 f.write("from juju.client import " + clients + "\n\n")
744 f.write(CLIENT_TABLE.format(clients=",\n ".join(
745 ['"{}": _client{}'.format(v, v) for v in captures])))
746 f.write(LOOKUP_FACADE)
747 f.write(TYPE_FACTORY)
748 for key in sorted([k for k in factories.keys() if "Facade" in k]):
749 print(factories[key], file=f)
750
751
752 def generate_facades(options):
753 captures = defaultdict(codegen.Capture)
754 schemas = {}
755 for p in sorted(glob(options.schema)):
756 if 'latest' in p:
757 juju_version = 'latest'
758 else:
759 try:
760 juju_version = re.search(JUJU_VERSION, p).group()
761 except AttributeError:
762 print("Cannot extract a juju version from {}".format(p))
763 print("Schemas must include a juju version in the filename")
764 raise SystemExit(1)
765
766 new_schemas = json.loads(Path(p).read_text("utf-8"))
767 schemas[juju_version] = [Schema(s) for s in new_schemas]
768
769 # Build all of the auxillary (unversioned) classes
770 # TODO: get rid of some of the excess trips through loops in the
771 # called functions.
772 for juju_version in sorted(schemas.keys()):
773 for schema in schemas[juju_version]:
774 schema.buildDefinitions()
775 buildTypes(schema, captures[schema.version])
776
777 # Build the Facade classes
778 for juju_version in sorted(schemas.keys()):
779 for schema in schemas[juju_version]:
780 cls, source = buildFacade(schema)
781 cls_name = "{}Facade".format(schema.name)
782
783 captures[schema.version].clear(cls_name)
784 # Make the factory class for _client.py
785 make_factory(cls_name)
786 # Make the actual class
787 captures[schema.version][cls_name].write(source)
788 # Build the methods for each Facade class.
789 buildMethods(cls, captures[schema.version])
790 # Mark this Facade class as being done for this version --
791 # helps mitigate some excessive looping.
792 CLASSES[schema.name] = cls
793
794 return captures
795
796
797 def setup():
798 parser = argparse.ArgumentParser()
799 parser.add_argument("-s", "--schema", default="juju/client/schemas*")
800 parser.add_argument("-o", "--output_dir", default="juju/client")
801 options = parser.parse_args()
802 return options
803
804
805 def main():
806 options = setup()
807
808 # Generate some text blobs
809 captures = generate_facades(options)
810
811 # ... and write them out
812 last_version = write_facades(captures, options)
813 write_definitions(captures, options, last_version)
814 write_client(captures, options)
815
816
817 if __name__ == '__main__':
818 main()