docs/find-missing.py

   1 #! /usr/bin/env python3
   2 # -*- coding: utf-8 -*-
   3
   4 # Copyright (c) 2019-2022. The SimGrid Team. All rights reserved.
   5
   6 # This program is free software; you can redistribute it and/or modify it
   7 # under the terms of the license (GNU LGPL) which comes with this package.
   8
   9 """
  10 Search for symbols documented in both the XML files produced by Doxygen and the python modules,
  11 but not documented with breathe in the RST files.
  12
  13 This script is tailored to SimGrid own needs.
  14
  15 If you are missing some dependencies, try:  pip3 install --requirement docs/requirements.txt
  16 """
  17
  18 import os
  19 import re
  20 import sys
  21 import xml.etree.ElementTree as ET
  22 import inspect
  23
  24 xml_files = [
  25     'build/xml/classsimgrid_1_1s4u_1_1Activity.xml',
  26     'build/xml/classsimgrid_1_1s4u_1_1Actor.xml',
  27     'build/xml/classsimgrid_1_1s4u_1_1Barrier.xml',
  28     'build/xml/classsimgrid_1_1s4u_1_1Comm.xml',
  29     'build/xml/classsimgrid_1_1s4u_1_1ConditionVariable.xml',
  30     'build/xml/classsimgrid_1_1s4u_1_1Disk.xml',
  31     'build/xml/classsimgrid_1_1s4u_1_1Engine.xml',
  32     'build/xml/classsimgrid_1_1s4u_1_1Exec.xml',
  33     'build/xml/classsimgrid_1_1s4u_1_1Host.xml',
  34     'build/xml/classsimgrid_1_1s4u_1_1Io.xml',
  35     'build/xml/classsimgrid_1_1s4u_1_1Link.xml',
  36     'build/xml/classsimgrid_1_1s4u_1_1Mailbox.xml',
  37     'build/xml/classsimgrid_1_1s4u_1_1Mutex.xml',
  38     'build/xml/classsimgrid_1_1s4u_1_1NetZone.xml',
  39     'build/xml/classsimgrid_1_1s4u_1_1Semaphore.xml',
  40     'build/xml/classsimgrid_1_1s4u_1_1VirtualMachine.xml',
  41     'build/xml/classsimgrid_1_1xbt_1_1signal_3_01R_07P_8_8_8_08_4.xml',
  42     'build/xml/namespacesimgrid_1_1s4u_1_1this__actor.xml',
  43     'build/xml/actor_8h.xml',
  44     'build/xml/barrier_8h.xml',
  45     'build/xml/cond_8h.xml',
  46     'build/xml/engine_8h.xml',
  47     'build/xml/forward_8h.xml',
  48     'build/xml/host_8h.xml',
  49     'build/xml/link_8h.xml',
  50     'build/xml/mailbox_8h.xml',
  51     'build/xml/msg_8h.xml',
  52     'build/xml/mutex_8h.xml',
  53     'build/xml/semaphore_8h.xml',
  54     'build/xml/vm_8h.xml',
  55     'build/xml/zone_8h.xml'
  56 ]
  57
  58 python_modules = [
  59     'simgrid'
  60 ]
  61 python_ignore = [
  62     'simgrid.ActorKilled'
  63 ]
  64
  65
  66 ############ Search the python elements in the source, and report undocumented ones
  67 ############
  68
  69 # data structure in which we store the declaration we find
  70 python_decl = {}
  71
  72 def handle_python_module(fullname, englobing, elm):
  73     """Recursive function exploring the python modules."""
  74
  75     def found_decl(kind, obj):
  76         """Helper function that add an object in the python_decl data structure"""
  77         if kind not in python_decl:
  78             python_decl[kind] = []
  79         python_decl[kind].append(obj)
  80
  81
  82     if fullname in python_ignore:
  83         print("Ignore Python symbol '{}' as requested.".format(fullname))
  84         return
  85
  86     if inspect.isroutine(elm) and inspect.isclass(englobing):
  87         found_decl("method", fullname)
  88 #        print('.. automethod:: {}'.format(fullname))
  89     elif inspect.isroutine(elm) and (not inspect.isclass(englobing)):
  90         found_decl("function", fullname)
  91 #        print('.. autofunction:: {}'.format(fullname))
  92     elif inspect.isdatadescriptor(elm):
  93         found_decl("attribute", fullname)
  94 #        print('.. autoattribute:: {}'.format(fullname))
  95     elif isinstance(elm, (int, str)): # We do have such a data, directly in the SimGrid top module
  96         found_decl("data", fullname)
  97 #        print('.. autodata:: {}'.format(fullname))
  98     elif inspect.ismodule(elm) or inspect.isclass(elm):
  99         for name, data in inspect.getmembers(elm):
 100             if name.startswith('__'):
 101                 continue
 102 #            print("Recurse on {}.{}".format(fullname, name))
 103             handle_python_module("{}.{}".format(fullname, name), elm, data)
 104     else:
 105         print('UNHANDLED TYPE {} : {!r} Type: {}'.format(fullname, elm, type(elm)))
 106
 107 # Start the recursion on the provided Python modules
 108 for name in python_modules:
 109     try:
 110         module = __import__(name)
 111     except Exception:
 112         if os.path.exists("../lib") and "../lib" not in sys.path:
 113             print("Adding ../lib to PYTHONPATH as {} cannot be imported".format(name))
 114             sys.path.append("../lib")
 115             try:
 116                 module = __import__(name)
 117             except Exception:
 118                 print("Cannot import {}, even with PYTHONPATH=../lib".format(name))
 119                 sys.exit(1)
 120         else:
 121             print("Cannot import {}".format(name))
 122             sys.exit(1)
 123     for sub in dir(module):
 124         if sub[0] == '_':
 125             continue
 126         handle_python_module("{}.{}".format(name, sub), module, getattr(module, sub))
 127
 128 # Forget about the python declarations that were actually done in the RST
 129 for kind in python_decl:
 130     with os.popen('grep \'[[:blank:]]*auto{}::\' source/*rst|sed \'s/^.*auto{}:: //\''.format(kind, kind)) as pse:
 131         for fullname in (l.strip() for l in pse):
 132             if fullname not in python_decl[kind]:
 133                 print("Warning: {} documented but declaration not found in python.".format(fullname))
 134             else:
 135                 python_decl[kind].remove(fullname)
 136 # Dump the missing ones
 137 for kind in python_decl:
 138     for fullname in python_decl[kind]:
 139         print(" .. auto{}:: {}".format(kind, fullname))
 140
 141 ################ And now deal with Doxygen declarations
 142 ################
 143
 144 doxy_funs = {} # {classname: {func_name: [args]} }
 145 doxy_vars = {} # {classname: [names]}
 146 doxy_type = {} # {classname: [names]}
 147
 148 # find the declarations in the XML files
 149 for arg in xml_files:
 150     if arg[-4:] != '.xml':
 151         print("Argument '{}' does not end with '.xml'".format(arg))
 152         continue
 153     #print("Parse file {}".format(arg))
 154     tree = ET.parse(arg)
 155     for elem in tree.findall(".//compounddef"):
 156         if elem.attrib["kind"] == "class":
 157             if elem.attrib["prot"] != "public":
 158                 continue
 159             if "compoundname" in elem:
 160                 raise Exception("Compound {} has no 'compoundname' child tag.".format(elem))
 161             compoundname = elem.find("compoundname").text
 162             #print("compoundname {}".format(compoundname))
 163         elif elem.attrib["kind"] == "file":
 164             compoundname = ""
 165         elif elem.attrib["kind"] == "namespace":
 166             compoundname = elem.find("compoundname").text
 167         else:
 168             print("Element {} is of kind {}".format(elem.attrib["id"], elem.attrib["kind"]))
 169
 170         for member in elem.findall('.//memberdef'):
 171             if member.attrib["prot"] != "public":
 172                 continue
 173             kind = member.attrib["kind"]
 174             name = member.find("name").text
 175             #print("kind:{} compoundname:{} name:{}".format( kind,compoundname, name))
 176             if kind == "variable":
 177                 if compoundname not in doxy_vars:
 178                     doxy_vars[compoundname] = []
 179                 doxy_vars[compoundname].append(name)
 180             elif kind == "function":
 181                 args = member.find('argsstring').text
 182                 args = re.sub(r'\)[^)]*$', ')', args) # ignore what's after the parameters (eg, '=0' or ' const')
 183
 184                 if compoundname not in doxy_funs:
 185                     doxy_funs[compoundname] = {}
 186                 if name not in doxy_funs[compoundname]:
 187                     doxy_funs[compoundname][name] = []
 188                 doxy_funs[compoundname][name].append(args)
 189             elif kind == "typedef":
 190                 if compoundname not in doxy_type:
 191                     doxy_type[compoundname] = []
 192                 doxy_type[compoundname].append(name)
 193             elif kind == "friend":
 194                 pass # Ignore friendship
 195             else:
 196                 print("member {}::{} is of kind {}".format(compoundname, name, kind))
 197
 198 # Forget about the declarations that are done in the RST
 199 with os.popen('grep doxygenfunction:: find-missing.ignore source/*rst|sed \'s/^.*doxygenfunction:: //\'|sed \'s/ *const//\'') as pse:
 200     for line in (l.strip() for l in pse):
 201         (klass, obj, args) = (None, None, None)
 202         if "(" in line:
 203             (line, args) = line.split('(', 1)
 204             args = "({}".format(args)
 205         if '::' in line:
 206             (klass, obj) = line.rsplit('::', 1)
 207         else:
 208             (klass, obj) = ("", line)
 209
 210         if klass not in doxy_funs:
 211             print("Warning: {} documented, but class {} not found in doxygen.".format(line, klass))
 212             continue
 213         if obj not in doxy_funs[klass]:
 214             print("Warning: Object '{}' documented but not found in '{}'".format(line, klass))
 215 #            for obj in doxy_funs[klass]:
 216 #                print("  found: {}::{}".format(klass, obj))
 217         elif len(doxy_funs[klass][obj]) == 1:
 218             del doxy_funs[klass][obj]
 219         elif args not in doxy_funs[klass][obj]:
 220             print("Warning: Function {}{} not found in {}".format(obj, args, klass))
 221         else:
 222             #print("Found {} in {}".format(line, klass))
 223             doxy_funs[klass][obj].remove(args)
 224             if len(doxy_funs[klass][obj]) == 0:
 225                 del doxy_funs[klass][obj]
 226 with os.popen('grep doxygenvariable:: find-missing.ignore source/*rst|sed \'s/^.*doxygenvariable:: //\'') as pse:
 227     for line in (l.strip() for l in pse):
 228         (klass, var) = line.rsplit('::', 1)
 229
 230         if klass not in doxy_vars:
 231             print("Warning: {} documented, but class {} not found in doxygen.".format(line, klass))
 232             continue
 233         if var not in doxy_vars[klass]:
 234             print("Warning: Object {} documented but not found in '{}'".format(line, klass))
 235         else:
 236 #            print("Found {} in {}".format(line, klass))
 237             doxy_vars[klass].remove(var)
 238             if len(doxy_vars[klass]) == 0:
 239                 del doxy_vars[klass]
 240 with os.popen('grep doxygentypedef:: find-missing.ignore source/*rst|sed \'s/^.*doxygentypedef:: //\'') as pse:
 241     for line in (l.strip() for l in pse):
 242         if '::' in line:
 243             (klass, typ) = line.rsplit('::', 1)
 244         else:
 245             (klass, typ) = ('', line)
 246
 247         if klass not in doxy_type:
 248             print("Warning: {} documented, but class {} not found in doxygen.".format(line, klass))
 249             continue
 250         if typ not in doxy_type[klass]:
 251             print("Warning: Type {} documented but not found in '{}'".format(line, klass))
 252         else:
 253 #            print("Found {} in {}".format(line, klass))
 254             doxy_type[klass].remove(typ)
 255             if len(doxy_type[klass]) == 0:
 256                 del doxy_type[klass]
 257
 258 # Dump the undocumented Doxygen declarations
 259 for obj in sorted(doxy_funs):
 260     for meth in sorted(doxy_funs[obj]):
 261         for args in sorted(doxy_funs[obj][meth]):
 262             if obj == '':
 263                 print(".. doxygenfunction:: {}{}".format(meth, args))
 264             else:
 265                 print(".. doxygenfunction:: {}::{}{}".format(obj, meth, args))
 266
 267 for obj in doxy_vars:
 268     for meth in sorted(doxy_vars[obj]):
 269         print(".. doxygenvariable:: {}::{}".format(obj, meth))
 270
 271 for obj in doxy_type:
 272     for meth in sorted(doxy_type[obj]):
 273         if obj == '':
 274             print(".. doxygentypedef:: {}".format(meth))
 275         else:
 276             print(".. doxygentypedef:: {}::{}".format(obj, meth))