docs/find-missing.py

   1 #! /usr/bin/env python3
   2 # -*- coding: utf-8 -*-
   3
   4 # Copyright (c) 2019-2021. The SimGrid Team. All rights reserved.
   5
   6 # This program is free software; you can redistribute it and/or modify it
   7 # under the terms of the license (GNU LGPL) which comes with this package.
   8
   9 """
  10 Search for symbols documented in both the XML files produced by Doxygen and the python modules,
  11 but not documented with breathe in the RST files.
  12
  13 This script is tailored to SimGrid own needs.
  14
  15 If you are missing some dependencies, try:  pip3 install --requirement docs/requirements.txt
  16 """
  17
  18 import fnmatch
  19 import os
  20 import re
  21 import sys
  22 import xml.etree.ElementTree as ET
  23 import inspect
  24
  25 xml_files = [
  26     'build/xml/classsimgrid_1_1s4u_1_1Activity.xml',
  27     'build/xml/classsimgrid_1_1s4u_1_1Actor.xml',
  28     'build/xml/classsimgrid_1_1s4u_1_1Barrier.xml',
  29     'build/xml/classsimgrid_1_1s4u_1_1Comm.xml',
  30     'build/xml/classsimgrid_1_1s4u_1_1ConditionVariable.xml',
  31     'build/xml/classsimgrid_1_1s4u_1_1Disk.xml',
  32     'build/xml/classsimgrid_1_1s4u_1_1Engine.xml',
  33     'build/xml/classsimgrid_1_1s4u_1_1Exec.xml',
  34     'build/xml/classsimgrid_1_1s4u_1_1Host.xml',
  35     'build/xml/classsimgrid_1_1s4u_1_1Io.xml',
  36     'build/xml/classsimgrid_1_1s4u_1_1Link.xml',
  37     'build/xml/classsimgrid_1_1s4u_1_1Mailbox.xml',
  38     'build/xml/classsimgrid_1_1s4u_1_1Mutex.xml',
  39     'build/xml/classsimgrid_1_1s4u_1_1NetZone.xml',
  40     'build/xml/classsimgrid_1_1s4u_1_1Semaphore.xml',
  41     'build/xml/classsimgrid_1_1s4u_1_1VirtualMachine.xml',
  42     'build/xml/classsimgrid_1_1xbt_1_1signal_3_01R_07P_8_8_8_08_4.xml',
  43     'build/xml/namespacesimgrid_1_1s4u_1_1this__actor.xml',
  44     'build/xml/actor_8h.xml',
  45     'build/xml/barrier_8h.xml',
  46     'build/xml/cond_8h.xml',
  47     'build/xml/engine_8h.xml',
  48     'build/xml/forward_8h.xml',
  49     'build/xml/host_8h.xml',
  50     'build/xml/link_8h.xml',
  51     'build/xml/mailbox_8h.xml',
  52     'build/xml/msg_8h.xml',
  53     'build/xml/mutex_8h.xml',
  54     'build/xml/semaphore_8h.xml',
  55     'build/xml/vm_8h.xml',
  56     'build/xml/zone_8h.xml'
  57 ]
  58
  59 python_modules = [
  60     'simgrid'
  61 ]
  62 python_ignore = [
  63     'simgrid.ActorKilled'
  64 ]
  65
  66
  67 ############ Search the python elements in the source, and report undocumented ones
  68 ############
  69
  70 # data structure in which we store the declaration we find
  71 python_decl = {}
  72
  73 def handle_python_module(fullname, englobing, elm):
  74     """Recursive function exploring the python modules."""
  75
  76     def found_decl(kind, obj):
  77         """Helper function that add an object in the python_decl data structure"""
  78         if kind not in python_decl:
  79             python_decl[kind] = []
  80         python_decl[kind].append(obj)
  81
  82
  83     if fullname in python_ignore:
  84         print ("Ignore Python symbol '{}' as requested.".format(fullname))
  85         return
  86
  87     if inspect.isroutine(elm) and inspect.isclass(englobing):
  88         found_decl("method", fullname)
  89 #        print('.. automethod:: {}'.format(fullname))
  90     elif inspect.isroutine(elm) and (not inspect.isclass(englobing)):
  91         found_decl("function", fullname)
  92 #        print('.. autofunction:: {}'.format(fullname))
  93     elif inspect.isdatadescriptor(elm):
  94         found_decl("attribute", fullname)
  95 #        print('.. autoattribute:: {}'.format(fullname))
  96     elif isinstance(elm, str) or isinstance(elm, int): # We do have such a data, directly in the SimGrid top module
  97         found_decl("data", fullname)
  98 #        print('.. autodata:: {}'.format(fullname))
  99     elif inspect.ismodule(elm) or inspect.isclass(elm):
 100         for name, data in inspect.getmembers(elm):
 101             if name.startswith('__'):
 102                 continue
 103 #            print("Recurse on {}.{}".format(fullname, name))
 104             handle_python_module("{}.{}".format(fullname, name), elm, data)
 105     else:
 106         print('UNHANDLED TYPE {} : {!r} Type: {}'.format(fullname, elm, type(elm)))
 107
 108 # Start the recursion on the provided Python modules
 109 for name in python_modules:
 110     try:
 111         module = __import__(name)
 112     except Exception:
 113         if os.path.exists("../lib") and "../lib" not in sys.path:
 114             print("Adding ../lib to PYTHONPATH as {} cannot be imported".format(name))
 115             sys.path.append("../lib")
 116             try:
 117                 module = __import__(name)
 118             except Exception:
 119                 print("Cannot import {}, even with PYTHONPATH=../lib".format(name))
 120                 sys.exit(1)
 121         else:
 122             print("Cannot import {}".format(name))
 123             sys.exit(1)
 124     for sub in dir(module):
 125         if sub[0] == '_':
 126             continue
 127         handle_python_module("{}.{}".format(name, sub), module, getattr(module, sub))
 128
 129 # Forget about the python declarations that were actually done in the RST
 130 for kind in python_decl:
 131     with os.popen('grep \'[[:blank:]]*auto{}::\' source/*rst|sed \'s/^.*auto{}:: //\''.format(kind, kind)) as pse:
 132         for fullname in (l.strip() for l in pse):
 133             if fullname not in python_decl[kind]:
 134                 print("Warning: {} documented but declaration not found in python.".format(fullname))
 135             else:
 136                 python_decl[kind].remove(fullname)
 137 # Dump the missing ones
 138 for kind in python_decl:
 139     for fullname in python_decl[kind]:
 140         print(" .. auto{}:: {}".format(kind, fullname))
 141
 142 ################ And now deal with Doxygen declarations
 143 ################
 144
 145 doxy_funs = {} # {classname: {func_name: [args]} }
 146 doxy_vars = {} # {classname: [names]}
 147 doxy_type = {} # {classname: [names]}
 148
 149 # find the declarations in the XML files
 150 for arg in xml_files:
 151     if arg[-4:] != '.xml':
 152         print ("Argument '{}' does not end with '.xml'".format(arg))
 153         continue
 154     #print("Parse file {}".format(arg))
 155     tree = ET.parse(arg)
 156     for elem in tree.findall(".//compounddef"):
 157         if elem.attrib["kind"] == "class":
 158             if elem.attrib["prot"] != "public":
 159                 continue
 160             if "compoundname" in elem:
 161                 raise Exception("Compound {} has no 'compoundname' child tag.".format(elem))
 162             compoundname = elem.find("compoundname").text
 163             #print ("compoundname {}".format(compoundname))
 164         elif elem.attrib["kind"] == "file":
 165             compoundname = ""
 166         elif elem.attrib["kind"] == "namespace":
 167             compoundname = elem.find("compoundname").text
 168         else:
 169             print("Element {} is of kind {}".format(elem.attrib["id"], elem.attrib["kind"]))
 170
 171         for member in elem.findall('.//memberdef'):
 172             if member.attrib["prot"] != "public":
 173                 continue
 174             kind = member.attrib["kind"]
 175             name = member.find("name").text
 176             #print("kind:{} compoundname:{} name:{}".format( kind,compoundname, name))
 177             if kind == "variable":
 178                 if compoundname not in doxy_vars:
 179                     doxy_vars[compoundname] = []
 180                 doxy_vars[compoundname].append(name)
 181             elif kind == "function":
 182                 args = member.find('argsstring').text
 183                 args = re.sub('\)[^)]*$', ')', args) # ignore what's after the parameters (eg, '=0' or ' const')
 184
 185                 if compoundname not in doxy_funs:
 186                     doxy_funs[compoundname] = {}
 187                 if name not in doxy_funs[compoundname]:
 188                     doxy_funs[compoundname][name] = []
 189                 doxy_funs[compoundname][name].append(args)
 190             elif kind == "typedef":
 191                 if compoundname not in doxy_type:
 192                     doxy_type[compoundname] = []
 193                 doxy_type[compoundname].append(name)
 194             elif kind == "friend":
 195                 pass # Ignore friendship
 196             else:
 197                 print ("member {}::{} is of kind {}".format(compoundname, name, kind))
 198
 199 # Forget about the declarations that are done in the RST
 200 with os.popen('grep doxygenfunction:: find-missing.ignore source/*rst|sed \'s/^.*doxygenfunction:: //\'|sed \'s/ *const//\'') as pse:
 201     for line in (l.strip() for l in pse):
 202         (klass, obj, args) = (None, None, None)
 203         if "(" in line:
 204             (line, args) = line.split('(', 1)
 205             args = "({}".format(args)
 206         if '::' in line:
 207             (klass, obj) = line.rsplit('::', 1)
 208         else:
 209             (klass, obj) = ("", line)
 210
 211         if klass not in doxy_funs:
 212             print("Warning: {} documented, but class {} not found in doxygen.".format(line, klass))
 213             continue
 214         if obj not in doxy_funs[klass]:
 215             print("Warning: Object '{}' documented but not found in '{}'".format(line, klass))
 216 #            for obj in doxy_funs[klass]:
 217 #                print("  found: {}::{}".format(klass, obj))
 218         elif len(doxy_funs[klass][obj])==1:
 219             del doxy_funs[klass][obj]
 220         elif args not in doxy_funs[klass][obj]:
 221             print("Warning: Function {}{} not found in {}".format(obj, args, klass))
 222         else:
 223             #print("Found {} in {}".format(line, klass))
 224             doxy_funs[klass][obj].remove(args)
 225             if len(doxy_funs[klass][obj]) == 0:
 226                 del doxy_funs[klass][obj]
 227 with os.popen('grep doxygenvariable:: find-missing.ignore source/*rst|sed \'s/^.*doxygenvariable:: //\'') as pse:
 228     for line in (l.strip() for l in pse):
 229         (klass, var) = line.rsplit('::', 1)
 230
 231         if klass not in doxy_vars:
 232             print("Warning: {} documented, but class {} not found in doxygen.".format(line, klass))
 233             continue
 234         if var not in doxy_vars[klass]:
 235             print("Warning: Object {} documented but not found in '{}'".format(line, klass))
 236         else:
 237 #            print("Found {} in {}".format(line, klass))
 238             doxy_vars[klass].remove(var)
 239             if len(doxy_vars[klass]) == 0:
 240                 del doxy_vars[klass]
 241 with os.popen('grep doxygentypedef:: find-missing.ignore source/*rst|sed \'s/^.*doxygentypedef:: //\'') as pse:
 242     for line in (l.strip() for l in pse):
 243         if '::' in line:
 244             (klass, typ) = line.rsplit('::', 1)
 245         else:
 246             (klass, typ) = ('', line)
 247
 248         if klass not in doxy_type:
 249             print("Warning: {} documented, but class {} not found in doxygen.".format(line, klass))
 250             continue
 251         if typ not in doxy_type[klass]:
 252             print("Warning: Type {} documented but not found in '{}'".format(line, klass))
 253         else:
 254 #            print("Found {} in {}".format(line, klass))
 255             doxy_type[klass].remove(typ)
 256             if len(doxy_type[klass]) == 0:
 257                 del doxy_type[klass]
 258
 259 # Dump the undocumented Doxygen declarations
 260 for obj in sorted(doxy_funs):
 261     for meth in sorted(doxy_funs[obj]):
 262         for args in sorted(doxy_funs[obj][meth]):
 263             if obj == '':
 264                 print(".. doxygenfunction:: {}{}".format(meth, args))
 265             else:
 266                 print(".. doxygenfunction:: {}::{}{}".format(obj, meth, args))
 267
 268 for obj in doxy_vars:
 269     for meth in sorted(doxy_vars[obj]):
 270         print(".. doxygenvariable:: {}::{}".format(obj, meth))
 271
 272 for obj in doxy_type:
 273     for meth in sorted(doxy_type[obj]):
 274         if obj == '':
 275             print(".. doxygentypedef:: {}".format(meth))
 276         else:
 277             print(".. doxygentypedef:: {}::{}".format(obj, meth))