juju/client/facade.py

   1 import argparse
   2 import builtins
   3 import functools
   4 import json
   5 import keyword
   6 from pathlib import Path
   7 import pprint
   8 import textwrap
   9 from typing import Sequence, Mapping, TypeVar, Any, Union
  10 import typing
  11
  12 from . import codegen
  13
  14 _marker = object()
  15
  16 # Map basic types to Python's typing with a callable
  17 SCHEMA_TO_PYTHON = {
  18     'string': str,
  19     'integer': int,
  20     'float': float,
  21     'number': float,
  22     'boolean': bool,
  23     'object': Any,
  24 }
  25
  26
  27 class KindRegistry(dict):
  28     def register(self, name, version, obj):
  29         self[name] = {version: {
  30             "object": obj,
  31         }}
  32
  33     def lookup(self, name, version=None):
  34         """If version is omitted, max version is used"""
  35         versions = self.get(name)
  36         if not versions:
  37             return None
  38         if version:
  39             return versions[version]
  40         return versions[max(versions)]
  41
  42     def getObj(self, name, version=None):
  43         result = self.lookup(name, version)
  44         if result:
  45             obj = result["object"]
  46             return obj
  47         return None
  48
  49
  50 class TypeRegistry(dict):
  51     def get(self, name):
  52         # Two way mapping
  53         refname = Schema.referenceName(name)
  54         if refname not in self:
  55             result = TypeVar(refname)
  56             self[refname] = result
  57             self[result] = refname
  58
  59         return self[refname]
  60
  61 _types = TypeRegistry()
  62 _registry = KindRegistry()
  63 classes = {}
  64
  65
  66 def booler(v):
  67     if isinstance(v, str):
  68         if v == "false":
  69             return False
  70     return bool(v)
  71
  72
  73 def getRefType(ref):
  74     return _types.get(ref)
  75
  76
  77 def refType(obj):
  78     return getRefType(obj["$ref"])
  79
  80
  81 def objType(obj):
  82     kind = obj.get('type')
  83     if not kind:
  84         raise ValueError("%s has no type" % obj)
  85     result = SCHEMA_TO_PYTHON.get(kind)
  86     if not result:
  87         raise ValueError("%s has type %s" % (obj, kind))
  88     return result
  89
  90
  91 basic_types = [str, bool, int, float]
  92
  93
  94 def name_to_py(name):
  95     result = name.replace("-", "_")
  96     result = result.lower()
  97     if keyword.iskeyword(result) or result in dir(builtins):
  98         result += "_"
  99     return result
 100
 101
 102 def strcast(kind, keep_builtins=False):
 103     if issubclass(kind, typing.GenericMeta):
 104         return str(kind)[1:]
 105     if str(kind).startswith('~'):
 106         return str(kind)[1:]
 107     if (kind in basic_types or
 108             type(kind) in basic_types) and keep_builtins is False:
 109         return kind.__name__
 110     return kind
 111
 112
 113 class Args(list):
 114     def __init__(self, defs):
 115         self.defs = defs
 116         if defs:
 117             rtypes = _registry.getObj(_types[defs])
 118             if len(rtypes) == 1:
 119                 if not self.do_explode(rtypes[0][1]):
 120                     for name, rtype in rtypes:
 121                         self.append((name, rtype))
 122             else:
 123                 for name, rtype in rtypes:
 124                     self.append((name, rtype))
 125
 126     def do_explode(self, kind):
 127         if kind in basic_types or type(kind) is typing.TypeVar:
 128             return False
 129         if not issubclass(kind, (typing.Sequence,
 130                                  typing.Mapping)):
 131             self.clear()
 132             self.extend(Args(kind))
 133             return True
 134         return False
 135
 136     def PyToSchemaMapping(self):
 137         m = {}
 138         for n, rt in self:
 139             m[name_to_py(n)] = n
 140         return m
 141
 142     def SchemaToPyMapping(self):
 143         m = {}
 144         for n, tr in self:
 145             m[n] = name_to_py(n)
 146         return m
 147
 148     def _format(self, name, rtype, typed=True):
 149         if typed:
 150             return "{} : {}".format(
 151                 name_to_py(name),
 152                 strcast(rtype)
 153             )
 154         else:
 155             return name_to_py(name)
 156
 157     def _get_arg_str(self, typed=False, joined=", "):
 158         if self:
 159             parts = []
 160             for item in self:
 161                 parts.append(self._format(item[0], item[1], typed))
 162             if joined:
 163                 return joined.join(parts)
 164             return parts
 165         return ''
 166
 167     def as_kwargs(self):
 168         if self:
 169             parts = []
 170             for item in self:
 171                 parts.append('{}=None'.format(name_to_py(item[0])))
 172             return ', '.join(parts)
 173         return ''
 174
 175     def typed(self):
 176         return self._get_arg_str(True)
 177
 178     def __str__(self):
 179         return self._get_arg_str(False)
 180
 181     def get_doc(self):
 182         return self._get_arg_str(True, "\n")
 183
 184
 185 def buildTypes(schema, capture):
 186     global classes
 187     INDENT = "    "
 188     for kind in sorted((k for k in _types if not isinstance(k, str)),
 189                        key=lambda x: str(x)):
 190         name = _types[kind]
 191         if name in classes:
 192             continue
 193         args = Args(kind)
 194         source = ["""
 195 class {}(Type):
 196     _toSchema = {}
 197     _toPy = {}
 198     def __init__(self{}{}):
 199         '''
 200 {}
 201         '''""".format(
 202             name,
 203             # pprint these to get stable ordering across regens
 204             pprint.pformat(args.PyToSchemaMapping(), width=999),
 205             pprint.pformat(args.SchemaToPyMapping(), width=999),
 206             ", " if args else "",
 207             args.as_kwargs(),
 208             textwrap.indent(args.get_doc(), INDENT * 2))
 209                   ]
 210         assignments = args._get_arg_str(False, False)
 211
 212         if not args:
 213             source.append("{}pass".format(INDENT * 2))
 214         else:
 215             for arg in args:
 216                 arg_name = name_to_py(arg[0])
 217                 arg_type = arg[1]
 218                 arg_type_name = strcast(arg_type)
 219                 if arg_type in basic_types:
 220                     source.append("{}self.{} = {}".format(INDENT * 2, arg_name, arg_name))
 221                 elif issubclass(arg_type, typing.Sequence):
 222                     value_type = (
 223                         arg_type_name.__parameters__[0]
 224                         if len(arg_type_name.__parameters__)
 225                         else None
 226                     )
 227                     if type(value_type) is typing.TypeVar:
 228                         source.append("{}self.{} = [{}.from_json(o) for o in {} or []]".format(
 229                             INDENT * 2, arg_name, strcast(value_type), arg_name))
 230                     else:
 231                         source.append("{}self.{} = {}".format(INDENT * 2, arg_name, arg_name))
 232                 elif issubclass(arg_type, typing.Mapping):
 233                     value_type = (
 234                         arg_type_name.__parameters__[1]
 235                         if len(arg_type_name.__parameters__) > 1
 236                         else None
 237                     )
 238                     if type(value_type) is typing.TypeVar:
 239                         source.append("{}self.{} = {{k: {}.from_json(v) for k, v in ({} or dict()).items()}}".format(
 240                             INDENT * 2, arg_name, strcast(value_type), arg_name))
 241                     else:
 242                         source.append("{}self.{} = {}".format(INDENT * 2, arg_name, arg_name))
 243                 elif type(arg_type) is typing.TypeVar:
 244                     source.append("{}self.{} = {}.from_json({}) if {} else None".format(
 245                         INDENT * 2, arg_name, arg_type_name, arg_name, arg_name))
 246                 else:
 247                     source.append("{}self.{} = {}".format(INDENT * 2, arg_name, arg_name))
 248
 249         source = "\n".join(source)
 250         capture.write(source)
 251         capture.write("\n\n")
 252         co = compile(source, __name__, "exec")
 253         ns = _getns()
 254         exec(co, ns)
 255         cls = ns[name]
 256         classes[name] = cls
 257
 258
 259 def retspec(defs):
 260     # return specs
 261     # only return 1, so if there is more than one type
 262     # we need to include a union
 263     # In truth there is only 1 return
 264     # Error or the expected Type
 265     if not defs:
 266         return None
 267     if defs in basic_types:
 268         return strcast(defs, False)
 269     rtypes = _registry.getObj(_types[defs])
 270     if not rtypes:
 271         return None
 272     if len(rtypes) > 1:
 273         return Union[tuple([strcast(r[1], True) for r in rtypes])]
 274     return strcast(rtypes[0][1], False)
 275
 276
 277 def return_type(defs):
 278     if not defs:
 279         return None
 280     rtypes = _registry.getObj(_types[defs])
 281     if not rtypes:
 282         return None
 283     if len(rtypes) > 1:
 284         for n, t in rtypes:
 285             if n == "Error":
 286                 continue
 287             return t
 288     return rtypes[0][1]
 289
 290
 291 def type_anno_func(func, defs, is_result=False):
 292     annos = {}
 293     if not defs:
 294         return func
 295     rtypes = _registry.getObj(_types[defs])
 296     if is_result:
 297         kn = "return"
 298         if not rtypes:
 299             annos[kn] = None
 300         elif len(rtypes) > 1:
 301             annos[kn] = Union[tuple([r[1] for r in rtypes])]
 302         else:
 303             annos[kn] = rtypes[0][1]
 304     else:
 305         for name, rtype in rtypes:
 306             name = name_to_py(name)
 307             annos[name] = rtype
 308     func.__annotations__.update(annos)
 309     return func
 310
 311
 312 def ReturnMapping(cls):
 313     # Annotate the method with a return Type
 314     # so the value can be cast
 315     def decorator(f):
 316         @functools.wraps(f)
 317         async def wrapper(*args, **kwargs):
 318             nonlocal cls
 319             reply = await f(*args, **kwargs)
 320             if cls is None:
 321                 return reply
 322             if 'error' in reply:
 323                 cls = classes['Error']
 324             if issubclass(cls, typing.Sequence):
 325                 result = []
 326                 item_cls = cls.__parameters__[0]
 327                 for item in reply:
 328                     result.append(item_cls.from_json(item))
 329                     """
 330                     if 'error' in item:
 331                         cls = classes['Error']
 332                     else:
 333                         cls = item_cls
 334                     result.append(cls.from_json(item))
 335                     """
 336             else:
 337                 result = cls.from_json(reply['response'])
 338
 339             return result
 340         return wrapper
 341     return decorator
 342
 343
 344 def makeFunc(cls, name, params, result, async=True):
 345     INDENT = "    "
 346     args = Args(params)
 347     assignments = []
 348     toschema = args.PyToSchemaMapping()
 349     for arg in args._get_arg_str(False, False):
 350         assignments.append("{}_params[\'{}\'] = {}".format(INDENT,
 351                                                            toschema[arg],
 352                                                            arg))
 353     assignments = "\n".join(assignments)
 354     res = retspec(result)
 355     source = """
 356
 357 @ReturnMapping({rettype})
 358 {async}def {name}(self{argsep}{args}):
 359     '''
 360 {docstring}
 361     Returns -> {res}
 362     '''
 363     # map input types to rpc msg
 364     _params = dict()
 365     msg = dict(type='{cls.name}', request='{name}', version={cls.version}, params=_params)
 366 {assignments}
 367     reply = {await}self.rpc(msg)
 368     return reply
 369
 370 """
 371
 372     fsource = source.format(async="async " if async else "",
 373                             name=name,
 374                             argsep=", " if args else "",
 375                             args=args,
 376                             res=res,
 377                             rettype=result.__name__ if result else None,
 378                             docstring=textwrap.indent(args.get_doc(), INDENT),
 379                             cls=cls,
 380                             assignments=assignments,
 381                             await="await " if async else "")
 382     ns = _getns()
 383     exec(fsource, ns)
 384     func = ns[name]
 385     return func, fsource
 386
 387
 388 def buildMethods(cls, capture):
 389     properties = cls.schema['properties']
 390     for methodname in sorted(properties):
 391         method, source = _buildMethod(cls, methodname)
 392         setattr(cls, methodname, method)
 393         capture.write(source, depth=1)
 394
 395
 396 def _buildMethod(cls, name):
 397     params = None
 398     result = None
 399     method = cls.schema['properties'][name]
 400     if 'properties' in method:
 401         prop = method['properties']
 402         spec = prop.get('Params')
 403         if spec:
 404             result = _types.get(spec['$ref'])
 405             if '$ref' in spec:
 406                 result = _types.get(spec['$ref'])
 407             else:
 408                 result = SCHEMA_TO_PYTHON[spec['type']]
 409         spec = prop.get('Result')
 410         if spec:
 411             if '$ref' in spec:
 412                 result = _types.get(spec['$ref'])
 413             else:
 414                 result = SCHEMA_TO_PYTHON[spec['type']]
 415     return makeFunc(cls, name, params, result)
 416
 417
 418 def buildFacade(schema):
 419     cls = type(schema.name, (Type,), dict(name=schema.name,
 420                                           version=schema.version,
 421                                           schema=schema))
 422     source = """
 423 class {name}Facade(Type):
 424     name = '{name}'
 425     version = {version}
 426     schema = {schema}
 427     """.format(name=schema.name,
 428                version=schema.version,
 429                schema=textwrap.indent(pprint.pformat(schema), "    "))
 430     return cls, source
 431
 432
 433 class TypeEncoder(json.JSONEncoder):
 434     def default(self, obj):
 435         if isinstance(obj, Type):
 436             return obj.serialize()
 437         return json.JSONEncoder.default(self, obj)
 438
 439
 440 class Type:
 441     def connect(self, connection):
 442         self.connection = connection
 443
 444     async def rpc(self, msg):
 445         result = await self.connection.rpc(msg, encoder=TypeEncoder)
 446         return result
 447
 448     @classmethod
 449     def from_json(cls, data):
 450         if isinstance(data, cls):
 451             return data
 452         if isinstance(data, str):
 453             data = json.loads(data)
 454         d = {}
 455         for k, v in (data or {}).items():
 456             d[cls._toPy.get(k, k)] = v
 457
 458         try:
 459             return cls(**d)
 460         except TypeError:
 461             raise
 462
 463     def serialize(self):
 464         d = {}
 465         for attr, tgt in self._toSchema.items():
 466             d[tgt] = getattr(self, attr)
 467         return d
 468
 469     def to_json(self):
 470         return json.dumps(self.serialize())
 471
 472
 473 class Schema(dict):
 474     def __init__(self, schema):
 475         self.name = schema['Name']
 476         self.version = schema['Version']
 477         self.update(schema['Schema'])
 478
 479     @classmethod
 480     def referenceName(cls, ref):
 481         if ref.startswith("#/definitions/"):
 482             ref = ref.rsplit("/", 1)[-1]
 483         return ref
 484
 485     def resolveDefinition(self, ref):
 486         return self['definitions'][self.referenceName(ref)]
 487
 488     def deref(self, prop, name):
 489         if not isinstance(prop, dict):
 490             raise TypeError(prop)
 491         if "$ref" not in prop:
 492             return prop
 493
 494         target = self.resolveDefinition(prop["$ref"])
 495         return target
 496
 497     def buildDefinitions(self):
 498         # here we are building the types out
 499         # anything in definitions is a type
 500         # but these may contain references themselves
 501         # so we dfs to the bottom and build upwards
 502         # when a types is already in the registry
 503         defs = self.get('definitions')
 504         if not defs:
 505             return
 506         for d, data in defs.items():
 507             if d in _registry:
 508                 continue
 509             node = self.deref(data, d)
 510             kind = node.get("type")
 511             if kind == "object":
 512                 result = self.buildObject(node, d)
 513             elif kind == "array":
 514                 pass
 515             _registry.register(d, self.version, result)
 516             # XXX: This makes sure that the type gets added to the global
 517             # _types dict even if no other type in the schema has a ref
 518             # to it.
 519             getRefType(d)
 520
 521     def buildObject(self, node, name=None, d=0):
 522         # we don't need to build types recursively here
 523         # they are all in definitions already
 524         # we only want to include the type reference
 525         # which we can derive from the name
 526         struct = []
 527         add = struct.append
 528         props = node.get("properties")
 529         pprops = node.get("patternProperties")
 530         if props:
 531             # Sort these so the __init__ arg list for each Type remains
 532             # consistently ordered across regens of client.py
 533             for p in sorted(props):
 534                 prop = props[p]
 535                 if "$ref" in prop:
 536                     add((p, refType(prop)))
 537                 else:
 538                     kind = prop['type']
 539                     if kind == "array":
 540                         add((p, self.buildArray(prop, d + 1)))
 541                     elif kind == "object":
 542                         struct.extend(self.buildObject(prop, p, d + 1))
 543                     else:
 544                         add((p, objType(prop)))
 545         if pprops:
 546             if ".*" not in pprops:
 547                 raise ValueError(
 548                     "Cannot handle actual pattern in patternProperties %s" %
 549                     pprops)
 550             pprop = pprops[".*"]
 551             if "$ref" in pprop:
 552                 add((name, Mapping[str, refType(pprop)]))
 553                 return struct
 554             ppkind = pprop["type"]
 555             if ppkind == "array":
 556                 add((name, self.buildArray(pprop, d + 1)))
 557             else:
 558                 add((name, Mapping[str, SCHEMA_TO_PYTHON[ppkind]]))
 559
 560         if not struct and node.get('additionalProperties', False):
 561             add((name, Mapping[str, SCHEMA_TO_PYTHON['object']]))
 562
 563         return struct
 564
 565     def buildArray(self, obj, d=0):
 566         # return a sequence from an array in the schema
 567         if "$ref" in obj:
 568             return Sequence[refType(obj)]
 569         else:
 570             kind = obj.get("type")
 571             if kind and kind == "array":
 572                 items = obj['items']
 573                 return self.buildArray(items, d + 1)
 574             else:
 575                 return Sequence[objType(obj)]
 576
 577
 578 def _getns():
 579     ns = {'Type': Type,
 580           'typing': typing,
 581           'ReturnMapping': ReturnMapping
 582           }
 583     # Copy our types into the globals of the method
 584     for facade in _registry:
 585         ns[facade] = _registry.getObj(facade)
 586     return ns
 587
 588
 589
 590 def generate_facacdes(options):
 591     global classes
 592     schemas = json.loads(Path(options.schema).read_text("utf-8"))
 593     capture = codegen.CodeWriter()
 594     capture.write(textwrap.dedent("""\
 595         # DO NOT CHANGE THIS FILE! This file is auto-generated by facade.py.
 596         # Changes will be overwritten/lost when the file is regenerated.
 597
 598         from juju.client.facade import Type, ReturnMapping
 599
 600     """))
 601     schemas = [Schema(s) for s in schemas]
 602
 603     for schema in schemas:
 604         schema.buildDefinitions()
 605         buildTypes(schema, capture)
 606
 607     for schema in schemas:
 608         # TODO generate class now with a metaclass that takes the schema
 609         # the generated class has the right name and it in turn uses
 610         # the metaclass to populate cls
 611         cls, source = buildFacade(schema)
 612         capture.write(source)
 613         buildMethods(cls, capture)
 614         classes[schema.name] = cls
 615
 616     return capture
 617
 618 def setup():
 619     parser = argparse.ArgumentParser()
 620     parser.add_argument("-s", "--schema", default="schemas.json")
 621     parser.add_argument("-o", "--output", default="client.py")
 622     options = parser.parse_args()
 623     return options
 624
 625 def main():
 626     options = setup()
 627     capture = generate_facacdes(options)
 628     with open(options.output, "w") as fp:
 629         print(capture, file=fp)
 630
 631
 632 if __name__ == '__main__':
 633     main()