c959e01649964a9a5eb955e2fc0829ebabfcc90f
[osm/N2VC.git] / modules / libjuju / juju / client / facade.py
1 import argparse
2 import builtins
3 from collections import defaultdict
4 import functools
5 from glob import glob
6 import json
7 import keyword
8 from pathlib import Path
9 import pprint
10 import re
11 import textwrap
12 from typing import Sequence, Mapping, TypeVar, Any, Union
13 import typing
14
15 from . import codegen
16
17 _marker = object()
18
19 JUJU_VERSION = re.compile('[0-9]+\.[0-9-]+[\.\-][0-9a-z]+(\.[0-9]+)?')
20 # Workaround for https://bugs.launchpad.net/juju/+bug/1683906
21 NAUGHTY_CLASSES = ['ClientFacade', 'Client', 'FullStatus', 'ModelStatusInfo',
22 'ModelInfo']
23
24
25 # Map basic types to Python's typing with a callable
26 SCHEMA_TO_PYTHON = {
27 'string': str,
28 'integer': int,
29 'float': float,
30 'number': float,
31 'boolean': bool,
32 'object': Any,
33 }
34
35
36 # Friendly warning message to stick at the top of generated files.
37 HEADER = """\
38 # DO NOT CHANGE THIS FILE! This file is auto-generated by facade.py.
39 # Changes will be overwritten/lost when the file is regenerated.
40
41 """
42
43
44 # Classes and helper functions that we'll write to _client.py
45 LOOKUP_FACADE = '''
46 def lookup_facade(name, version):
47 """
48 Given a facade name and version, attempt to pull that facade out
49 of the correct client<version>.py file.
50
51 """
52 try:
53 facade = getattr(CLIENTS[str(version)], name)
54 except KeyError:
55 raise ImportError("No facades found for version {}".format(version))
56 except AttributeError:
57 raise ImportError(
58 "No facade with name '{}' in version {}".format(name, version))
59 return facade
60
61
62 '''
63
64 TYPE_FACTORY = '''
65 class TypeFactory:
66 @classmethod
67 def from_connection(cls, connection):
68 """
69 Given a connected Connection object, return an initialized and
70 connected instance of an API Interface matching the name of
71 this class.
72
73 @param connection: initialized Connection object.
74
75 """
76 version = connection.facades[cls.__name__[:-6]]
77
78 c = lookup_facade(cls.__name__, version)
79 c = c()
80 c.connect(connection)
81
82 return c
83
84
85 '''
86
87 CLIENT_TABLE = '''
88 CLIENTS = {{
89 {clients}
90 }}
91
92
93 '''
94
95
96 class KindRegistry(dict):
97 def register(self, name, version, obj):
98 self[name] = {version: {
99 "object": obj,
100 }}
101
102 def lookup(self, name, version=None):
103 """If version is omitted, max version is used"""
104 versions = self.get(name)
105 if not versions:
106 return None
107 if version:
108 return versions[version]
109 return versions[max(versions)]
110
111 def getObj(self, name, version=None):
112 result = self.lookup(name, version)
113 if result:
114 obj = result["object"]
115 return obj
116 return None
117
118
119 class TypeRegistry(dict):
120 def get(self, name):
121 # Two way mapping
122 refname = Schema.referenceName(name)
123 if refname not in self:
124 result = TypeVar(refname)
125 self[refname] = result
126 self[result] = refname
127
128 return self[refname]
129
130 _types = TypeRegistry()
131 _registry = KindRegistry()
132 CLASSES = {}
133 factories = codegen.Capture()
134
135
136 def booler(v):
137 if isinstance(v, str):
138 if v == "false":
139 return False
140 return bool(v)
141
142
143 def getRefType(ref):
144 return _types.get(ref)
145
146
147 def refType(obj):
148 return getRefType(obj["$ref"])
149
150
151 def objType(obj):
152 kind = obj.get('type')
153 if not kind:
154 raise ValueError("%s has no type" % obj)
155 result = SCHEMA_TO_PYTHON.get(kind)
156 if not result:
157 raise ValueError("%s has type %s" % (obj, kind))
158 return result
159
160
161 basic_types = [str, bool, int, float]
162
163
164 def name_to_py(name):
165 result = name.replace("-", "_")
166 result = result.lower()
167 if keyword.iskeyword(result) or result in dir(builtins):
168 result += "_"
169 return result
170
171
172 def strcast(kind, keep_builtins=False):
173 if issubclass(kind, typing.GenericMeta):
174 return str(kind)[1:]
175 if str(kind).startswith('~'):
176 return str(kind)[1:]
177 if (kind in basic_types or
178 type(kind) in basic_types) and keep_builtins is False:
179 return kind.__name__
180 return kind
181
182
183 class Args(list):
184 def __init__(self, defs):
185 self.defs = defs
186 if defs:
187 rtypes = _registry.getObj(_types[defs])
188 if len(rtypes) == 1:
189 if not self.do_explode(rtypes[0][1]):
190 for name, rtype in rtypes:
191 self.append((name, rtype))
192 else:
193 for name, rtype in rtypes:
194 self.append((name, rtype))
195
196 def do_explode(self, kind):
197 if kind in basic_types or type(kind) is typing.TypeVar:
198 return False
199 if not issubclass(kind, (typing.Sequence,
200 typing.Mapping)):
201 self.clear()
202 self.extend(Args(kind))
203 return True
204 return False
205
206 def PyToSchemaMapping(self):
207 m = {}
208 for n, rt in self:
209 m[name_to_py(n)] = n
210 return m
211
212 def SchemaToPyMapping(self):
213 m = {}
214 for n, tr in self:
215 m[n] = name_to_py(n)
216 return m
217
218 def _format(self, name, rtype, typed=True):
219 if typed:
220 return "{} : {}".format(
221 name_to_py(name),
222 strcast(rtype)
223 )
224 else:
225 return name_to_py(name)
226
227 def _get_arg_str(self, typed=False, joined=", "):
228 if self:
229 parts = []
230 for item in self:
231 parts.append(self._format(item[0], item[1], typed))
232 if joined:
233 return joined.join(parts)
234 return parts
235 return ''
236
237 def as_kwargs(self):
238 if self:
239 parts = []
240 for item in self:
241 parts.append('{}=None'.format(name_to_py(item[0])))
242 return ', '.join(parts)
243 return ''
244
245 def typed(self):
246 return self._get_arg_str(True)
247
248 def __str__(self):
249 return self._get_arg_str(False)
250
251 def get_doc(self):
252 return self._get_arg_str(True, "\n")
253
254
255 def buildTypes(schema, capture):
256 INDENT = " "
257 for kind in sorted((k for k in _types if not isinstance(k, str)),
258 key=lambda x: str(x)):
259 name = _types[kind]
260 if name in capture and not name in NAUGHTY_CLASSES:
261 continue
262 args = Args(kind)
263 # Write Factory class for _client.py
264 make_factory(name)
265 # Write actual class
266 source = ["""
267 class {}(Type):
268 _toSchema = {}
269 _toPy = {}
270 def __init__(self{}{}, **unknown_fields):
271 '''
272 {}
273 '''""".format(
274 name,
275 # pprint these to get stable ordering across regens
276 pprint.pformat(args.PyToSchemaMapping(), width=999),
277 pprint.pformat(args.SchemaToPyMapping(), width=999),
278 ", " if args else "",
279 args.as_kwargs(),
280 textwrap.indent(args.get_doc(), INDENT * 2))
281 ]
282 assignments = args._get_arg_str(False, False)
283
284 if not args:
285 source.append("{}pass".format(INDENT * 2))
286 else:
287 for arg in args:
288 arg_name = name_to_py(arg[0])
289 arg_type = arg[1]
290 arg_type_name = strcast(arg_type)
291 if arg_type in basic_types:
292 source.append("{}self.{} = {}".format(INDENT * 2, arg_name, arg_name))
293 elif issubclass(arg_type, typing.Sequence):
294 value_type = (
295 arg_type_name.__parameters__[0]
296 if len(arg_type_name.__parameters__)
297 else None
298 )
299 if type(value_type) is typing.TypeVar:
300 source.append("{}self.{} = [{}.from_json(o) for o in {} or []]".format(
301 INDENT * 2, arg_name, strcast(value_type), arg_name))
302 else:
303 source.append("{}self.{} = {}".format(INDENT * 2, arg_name, arg_name))
304 elif issubclass(arg_type, typing.Mapping):
305 value_type = (
306 arg_type_name.__parameters__[1]
307 if len(arg_type_name.__parameters__) > 1
308 else None
309 )
310 if type(value_type) is typing.TypeVar:
311 source.append("{}self.{} = {{k: {}.from_json(v) for k, v in ({} or dict()).items()}}".format(
312 INDENT * 2, arg_name, strcast(value_type), arg_name))
313 else:
314 source.append("{}self.{} = {}".format(INDENT * 2, arg_name, arg_name))
315 elif type(arg_type) is typing.TypeVar:
316 source.append("{}self.{} = {}.from_json({}) if {} else None".format(
317 INDENT * 2, arg_name, arg_type_name, arg_name, arg_name))
318 else:
319 source.append("{}self.{} = {}".format(INDENT * 2, arg_name, arg_name))
320
321 source = "\n".join(source)
322 capture.clear(name)
323 capture[name].write(source)
324 capture[name].write("\n\n")
325 co = compile(source, __name__, "exec")
326 ns = _getns()
327 exec(co, ns)
328 cls = ns[name]
329 CLASSES[name] = cls
330
331
332 def retspec(defs):
333 # return specs
334 # only return 1, so if there is more than one type
335 # we need to include a union
336 # In truth there is only 1 return
337 # Error or the expected Type
338 if not defs:
339 return None
340 if defs in basic_types:
341 return strcast(defs, False)
342 rtypes = _registry.getObj(_types[defs])
343 if not rtypes:
344 return None
345 if len(rtypes) > 1:
346 return Union[tuple([strcast(r[1], True) for r in rtypes])]
347 return strcast(rtypes[0][1], False)
348
349
350 def return_type(defs):
351 if not defs:
352 return None
353 rtypes = _registry.getObj(_types[defs])
354 if not rtypes:
355 return None
356 if len(rtypes) > 1:
357 for n, t in rtypes:
358 if n == "Error":
359 continue
360 return t
361 return rtypes[0][1]
362
363
364 def type_anno_func(func, defs, is_result=False):
365 annos = {}
366 if not defs:
367 return func
368 rtypes = _registry.getObj(_types[defs])
369 if is_result:
370 kn = "return"
371 if not rtypes:
372 annos[kn] = None
373 elif len(rtypes) > 1:
374 annos[kn] = Union[tuple([r[1] for r in rtypes])]
375 else:
376 annos[kn] = rtypes[0][1]
377 else:
378 for name, rtype in rtypes:
379 name = name_to_py(name)
380 annos[name] = rtype
381 func.__annotations__.update(annos)
382 return func
383
384
385 def ReturnMapping(cls):
386 # Annotate the method with a return Type
387 # so the value can be cast
388 def decorator(f):
389 @functools.wraps(f)
390 async def wrapper(*args, **kwargs):
391 nonlocal cls
392 reply = await f(*args, **kwargs)
393 if cls is None:
394 return reply
395 if 'error' in reply:
396 cls = CLASSES['Error']
397 if issubclass(cls, typing.Sequence):
398 result = []
399 item_cls = cls.__parameters__[0]
400 for item in reply:
401 result.append(item_cls.from_json(item))
402 """
403 if 'error' in item:
404 cls = CLASSES['Error']
405 else:
406 cls = item_cls
407 result.append(cls.from_json(item))
408 """
409 else:
410 result = cls.from_json(reply['response'])
411
412 return result
413 return wrapper
414 return decorator
415
416
417 def makeFunc(cls, name, params, result, async=True):
418 INDENT = " "
419 args = Args(params)
420 assignments = []
421 toschema = args.PyToSchemaMapping()
422 for arg in args._get_arg_str(False, False):
423 assignments.append("{}_params[\'{}\'] = {}".format(INDENT,
424 toschema[arg],
425 arg))
426 assignments = "\n".join(assignments)
427 res = retspec(result)
428 source = """
429
430 @ReturnMapping({rettype})
431 {async}def {name}(self{argsep}{args}):
432 '''
433 {docstring}
434 Returns -> {res}
435 '''
436 # map input types to rpc msg
437 _params = dict()
438 msg = dict(type='{cls.name}', request='{name}', version={cls.version}, params=_params)
439 {assignments}
440 reply = {await}self.rpc(msg)
441 return reply
442
443 """
444
445 fsource = source.format(async="async " if async else "",
446 name=name,
447 argsep=", " if args else "",
448 args=args,
449 res=res,
450 rettype=result.__name__ if result else None,
451 docstring=textwrap.indent(args.get_doc(), INDENT),
452 cls=cls,
453 assignments=assignments,
454 await="await " if async else "")
455 ns = _getns()
456 exec(fsource, ns)
457 func = ns[name]
458 return func, fsource
459
460
461 def buildMethods(cls, capture):
462 properties = cls.schema['properties']
463 for methodname in sorted(properties):
464 method, source = _buildMethod(cls, methodname)
465 setattr(cls, methodname, method)
466 capture["{}Facade".format(cls.__name__)].write(source, depth=1)
467
468
469 def _buildMethod(cls, name):
470 params = None
471 result = None
472 method = cls.schema['properties'][name]
473 if 'properties' in method:
474 prop = method['properties']
475 spec = prop.get('Params')
476 if spec:
477 params = _types.get(spec['$ref'])
478 spec = prop.get('Result')
479 if spec:
480 if '$ref' in spec:
481 result = _types.get(spec['$ref'])
482 else:
483 result = SCHEMA_TO_PYTHON[spec['type']]
484 return makeFunc(cls, name, params, result)
485
486
487 def buildFacade(schema):
488 cls = type(schema.name, (Type,), dict(name=schema.name,
489 version=schema.version,
490 schema=schema))
491 source = """
492 class {name}Facade(Type):
493 name = '{name}'
494 version = {version}
495 schema = {schema}
496 """.format(name=schema.name,
497 version=schema.version,
498 schema=textwrap.indent(pprint.pformat(schema), " "))
499 return cls, source
500
501
502 class TypeEncoder(json.JSONEncoder):
503 def default(self, obj):
504 if isinstance(obj, Type):
505 return obj.serialize()
506 return json.JSONEncoder.default(self, obj)
507
508
509 class Type:
510 def connect(self, connection):
511 self.connection = connection
512
513 async def rpc(self, msg):
514 result = await self.connection.rpc(msg, encoder=TypeEncoder)
515 return result
516
517 @classmethod
518 def from_json(cls, data):
519 if isinstance(data, cls):
520 return data
521 if isinstance(data, str):
522 try:
523 data = json.loads(data)
524 except json.JSONDecodeError:
525 raise
526 d = {}
527 for k, v in (data or {}).items():
528 d[cls._toPy.get(k, k)] = v
529
530 try:
531 return cls(**d)
532 except TypeError:
533 raise
534
535 def serialize(self):
536 d = {}
537 for attr, tgt in self._toSchema.items():
538 d[tgt] = getattr(self, attr)
539 return d
540
541 def to_json(self):
542 return json.dumps(self.serialize())
543
544
545 class Schema(dict):
546 def __init__(self, schema):
547 self.name = schema['Name']
548 self.version = schema['Version']
549 self.update(schema['Schema'])
550
551 @classmethod
552 def referenceName(cls, ref):
553 if ref.startswith("#/definitions/"):
554 ref = ref.rsplit("/", 1)[-1]
555 return ref
556
557 def resolveDefinition(self, ref):
558 return self['definitions'][self.referenceName(ref)]
559
560 def deref(self, prop, name):
561 if not isinstance(prop, dict):
562 raise TypeError(prop)
563 if "$ref" not in prop:
564 return prop
565
566 target = self.resolveDefinition(prop["$ref"])
567 return target
568
569 def buildDefinitions(self):
570 # here we are building the types out
571 # anything in definitions is a type
572 # but these may contain references themselves
573 # so we dfs to the bottom and build upwards
574 # when a types is already in the registry
575 defs = self.get('definitions')
576 if not defs:
577 return
578 for d, data in defs.items():
579 if d in _registry and not d in NAUGHTY_CLASSES:
580 continue
581 node = self.deref(data, d)
582 kind = node.get("type")
583 if kind == "object":
584 result = self.buildObject(node, d)
585 elif kind == "array":
586 pass
587 _registry.register(d, self.version, result)
588 # XXX: This makes sure that the type gets added to the global
589 # _types dict even if no other type in the schema has a ref
590 # to it.
591 getRefType(d)
592
593 def buildObject(self, node, name=None, d=0):
594 # we don't need to build types recursively here
595 # they are all in definitions already
596 # we only want to include the type reference
597 # which we can derive from the name
598 struct = []
599 add = struct.append
600 props = node.get("properties")
601 pprops = node.get("patternProperties")
602 if props:
603 # Sort these so the __init__ arg list for each Type remains
604 # consistently ordered across regens of client.py
605 for p in sorted(props):
606 prop = props[p]
607 if "$ref" in prop:
608 add((p, refType(prop)))
609 else:
610 kind = prop['type']
611 if kind == "array":
612 add((p, self.buildArray(prop, d + 1)))
613 elif kind == "object":
614 struct.extend(self.buildObject(prop, p, d + 1))
615 else:
616 add((p, objType(prop)))
617 if pprops:
618 if ".*" not in pprops:
619 raise ValueError(
620 "Cannot handle actual pattern in patternProperties %s" %
621 pprops)
622 pprop = pprops[".*"]
623 if "$ref" in pprop:
624 add((name, Mapping[str, refType(pprop)]))
625 return struct
626 ppkind = pprop["type"]
627 if ppkind == "array":
628 add((name, self.buildArray(pprop, d + 1)))
629 else:
630 add((name, Mapping[str, SCHEMA_TO_PYTHON[ppkind]]))
631
632 if not struct and node.get('additionalProperties', False):
633 add((name, Mapping[str, SCHEMA_TO_PYTHON['object']]))
634
635 return struct
636
637 def buildArray(self, obj, d=0):
638 # return a sequence from an array in the schema
639 if "$ref" in obj:
640 return Sequence[refType(obj)]
641 else:
642 kind = obj.get("type")
643 if kind and kind == "array":
644 items = obj['items']
645 return self.buildArray(items, d + 1)
646 else:
647 return Sequence[objType(obj)]
648
649
650 def _getns():
651 ns = {'Type': Type,
652 'typing': typing,
653 'ReturnMapping': ReturnMapping
654 }
655 # Copy our types into the globals of the method
656 for facade in _registry:
657 ns[facade] = _registry.getObj(facade)
658 return ns
659
660
661 def make_factory(name):
662 if name in factories:
663 del factories[name]
664 factories[name].write("class {}(TypeFactory):\n pass\n\n".format(name))
665
666
667 def write_facades(captures, options):
668 """
669 Write the Facades to the appropriate _client<version>.py
670
671 """
672 for version in sorted(captures.keys()):
673 filename = "{}/_client{}.py".format(options.output_dir, version)
674 with open(filename, "w") as f:
675 f.write(HEADER)
676 f.write("from juju.client.facade import Type, ReturnMapping\n")
677 f.write("from juju.client._definitions import *\n\n")
678 for key in sorted(
679 [k for k in captures[version].keys() if "Facade" in k]):
680 print(captures[version][key], file=f)
681
682 # Return the last (most recent) version for use in other routines.
683 return version
684
685
686 def write_definitions(captures, options, version):
687 """
688 Write auxillary (non versioned) classes to
689 _definitions.py The auxillary classes currently get
690 written redudantly into each capture object, so we can look in
691 one of them -- we just use the last one from the loop above.
692
693 """
694 with open("{}/_definitions.py".format(options.output_dir), "w") as f:
695 f.write(HEADER)
696 f.write("from juju.client.facade import Type, ReturnMapping\n\n")
697 for key in sorted(
698 [k for k in captures[version].keys() if "Facade" not in k]):
699 print(captures[version][key], file=f)
700
701
702 def write_client(captures, options):
703 """
704 Write the TypeFactory classes to _client.py, along with some
705 imports and tables so that we can look up versioned Facades.
706
707 """
708 with open("{}/_client.py".format(options.output_dir), "w") as f:
709 f.write(HEADER)
710 f.write("from juju.client._definitions import *\n\n")
711 clients = ", ".join("_client{}".format(v) for v in captures)
712 f.write("from juju.client import " + clients + "\n\n")
713 f.write(CLIENT_TABLE.format(clients=",\n ".join(
714 ['"{}": _client{}'.format(v, v) for v in captures])))
715 f.write(LOOKUP_FACADE)
716 f.write(TYPE_FACTORY)
717 for key in sorted([k for k in factories.keys() if "Facade" in k]):
718 print(factories[key], file=f)
719
720
721 def generate_facades(options):
722 captures = defaultdict(codegen.Capture)
723 schemas = {}
724 for p in sorted(glob(options.schema)):
725 if 'latest' in p:
726 juju_version = 'latest'
727 else:
728 try:
729 juju_version = re.search(JUJU_VERSION, p).group()
730 except AttributeError:
731 print("Cannot extract a juju version from {}".format(p))
732 print("Schemas must include a juju version in the filename")
733 raise SystemExit(1)
734
735 new_schemas = json.loads(Path(p).read_text("utf-8"))
736 schemas[juju_version] = [Schema(s) for s in new_schemas]
737
738 # Build all of the auxillary (unversioned) classes
739 # TODO: get rid of some of the excess trips through loops in the
740 # called functions.
741 for juju_version in sorted(schemas.keys()):
742 for schema in schemas[juju_version]:
743 schema.buildDefinitions()
744 buildTypes(schema, captures[schema.version])
745
746 # Build the Facade classes
747 for juju_version in sorted(schemas.keys()):
748 for schema in schemas[juju_version]:
749 cls, source = buildFacade(schema)
750 cls_name = "{}Facade".format(schema.name)
751
752 captures[schema.version].clear(cls_name)
753 # Make the factory class for _client.py
754 make_factory(cls_name)
755 # Make the actual class
756 captures[schema.version][cls_name].write(source)
757 # Build the methods for each Facade class.
758 buildMethods(cls, captures[schema.version])
759 # Mark this Facade class as being done for this version --
760 # helps mitigate some excessive looping.
761 CLASSES[schema.name] = cls
762
763 return captures
764
765 def setup():
766 parser = argparse.ArgumentParser()
767 parser.add_argument("-s", "--schema", default="juju/client/schemas*")
768 parser.add_argument("-o", "--output_dir", default="juju/client")
769 options = parser.parse_args()
770 return options
771
772 def main():
773 options = setup()
774
775 # Generate some text blobs
776 captures = generate_facades(options)
777
778 # ... and write them out
779 last_version = write_facades(captures, options)
780 write_definitions(captures, options, last_version)
781 write_client(captures, options)
782
783 if __name__ == '__main__':
784 main()