docs/find-missing.py

   1 #! /usr/bin/env python3
   2 # -*- coding: utf-8 -*-
   3
   4 # Copyright (c) 2019-2021. The SimGrid Team. All rights reserved.
   5
   6 # This program is free software; you can redistribute it and/or modify it
   7 # under the terms of the license (GNU LGPL) which comes with this package.
   8
   9 """
  10 Search for symbols documented in both the XML files produced by Doxygen and the python modules,
  11 but not documented with breathe in the RST files.
  12
  13 This script is tailored to SimGrid own needs.
  14
  15 If you are missing some dependencies, try:  pip3 install --requirement docs/requirements.txt
  16 """
  17
  18 import fnmatch
  19 import os
  20 import re
  21 import sys
  22 import xml.etree.ElementTree as ET
  23 import inspect
  24
  25 xml_files = [
  26     'build/xml/classsimgrid_1_1s4u_1_1Activity.xml',
  27     'build/xml/classsimgrid_1_1s4u_1_1Actor.xml',
  28     'build/xml/classsimgrid_1_1s4u_1_1Barrier.xml',
  29     'build/xml/classsimgrid_1_1s4u_1_1Comm.xml',
  30     'build/xml/classsimgrid_1_1s4u_1_1ConditionVariable.xml',
  31     'build/xml/classsimgrid_1_1s4u_1_1Disk.xml',
  32     'build/xml/classsimgrid_1_1s4u_1_1Engine.xml',
  33     'build/xml/classsimgrid_1_1s4u_1_1Exec.xml',
  34     'build/xml/classsimgrid_1_1s4u_1_1Host.xml',
  35     'build/xml/classsimgrid_1_1s4u_1_1Io.xml',
  36     'build/xml/classsimgrid_1_1s4u_1_1Link.xml',
  37     'build/xml/classsimgrid_1_1s4u_1_1Mailbox.xml',
  38     'build/xml/classsimgrid_1_1s4u_1_1Mutex.xml',
  39     'build/xml/classsimgrid_1_1s4u_1_1NetZone.xml',
  40     'build/xml/classsimgrid_1_1s4u_1_1Semaphore.xml',
  41     'build/xml/classsimgrid_1_1s4u_1_1VirtualMachine.xml',
  42     'build/xml/classsimgrid_1_1xbt_1_1signal_3_01R_07P_8_8_8_08_4.xml',
  43     'build/xml/namespacesimgrid_1_1s4u_1_1this__actor.xml',
  44     'build/xml/actor_8h.xml',
  45     'build/xml/barrier_8h.xml',
  46     'build/xml/cond_8h.xml',
  47     'build/xml/engine_8h.xml',
  48     'build/xml/forward_8h.xml',
  49     'build/xml/host_8h.xml',
  50     'build/xml/link_8h.xml',
  51     'build/xml/mailbox_8h.xml',
  52     'build/xml/msg_8h.xml',
  53     'build/xml/mutex_8h.xml',
  54     'build/xml/semaphore_8h.xml',
  55     'build/xml/vm_8h.xml',
  56     'build/xml/zone_8h.xml'
  57 ]
  58
  59 python_modules = [
  60     'simgrid'
  61 ]
  62 python_ignore = [
  63     'simgrid.ActorKilled'
  64 ]
  65
  66
  67 ############ Search the python elements in the source, and report undocumented ones
  68 ############
  69
  70 # data structure in which we store the declaration we find
  71 python_decl = {}
  72
  73 def handle_python_module(fullname, englobing, elm):
  74     """Recursive function exploring the python modules."""
  75
  76     def found_decl(kind, obj):
  77         """Helper function that add an object in the python_decl data structure"""
  78         if kind not in python_decl:
  79             python_decl[kind] = []
  80         python_decl[kind].append(obj)
  81
  82
  83     if fullname in python_ignore:
  84         print ("Ignore Python symbol '{}' as requested.".format(fullname))
  85         return
  86
  87     if inspect.isroutine(elm) and inspect.isclass(englobing):
  88         found_decl("method", fullname)
  89 #        print('.. automethod:: {}'.format(fullname))
  90     elif inspect.isroutine(elm) and (not inspect.isclass(englobing)):
  91         found_decl("function", fullname)
  92 #        print('.. autofunction:: {}'.format(fullname))
  93     elif inspect.isdatadescriptor(elm):
  94         found_decl("attribute", fullname)
  95 #        print('.. autoattribute:: {}'.format(fullname))
  96     elif isinstance(elm, str) or isinstance(elm, int): # We do have such a data, directly in the SimGrid top module
  97         found_decl("data", fullname)
  98 #        print('.. autodata:: {}'.format(fullname))
  99     elif inspect.ismodule(elm) or inspect.isclass(elm):
 100         for name, data in inspect.getmembers(elm):
 101             if name.startswith('__'):
 102                 continue
 103 #            print("Recurse on {}.{}".format(fullname, name))
 104             handle_python_module("{}.{}".format(fullname, name), elm, data)
 105     else:
 106         print('UNHANDLED TYPE {} : {!r} Type: {}'.format(fullname, elm, type(elm)))
 107
 108 # Start the recursion on the provided Python modules
 109 for name in python_modules:
 110     try:
 111         module = __import__(name)
 112     except Exception:
 113         if os.path.exists("../lib") and "../lib" not in sys.path:
 114             print("Adding ../lib to PYTHONPATH as {} cannot be imported".format(name))
 115             sys.path.append("../lib")
 116             try:
 117                 module = __import__(name)
 118             except Exception:
 119                 print("Cannot import {}, even with PYTHONPATH=../lib".format(name))
 120                 sys.exit(1)
 121         else:
 122             print("Cannot import {}".format(name))
 123             sys.exit(1)
 124     for sub in dir(module):
 125         if sub[0] == '_':
 126             continue
 127         handle_python_module("{}.{}".format(name, sub), module, getattr(module, sub))
 128
 129 # Forget about the python declarations that were actually done in the RST
 130 for kind in python_decl:
 131     with os.popen('grep \'[[:blank:]]*auto{}::\' source/*rst|sed \'s/^.*auto{}:: //\''.format(kind, kind)) as pse:
 132         for fullname in (l.strip() for l in pse):
 133             if fullname not in python_decl[kind]:
 134                 print("Warning: {} documented but declaration not found in python.".format(fullname))
 135             else:
 136                 python_decl[kind].remove(fullname)
 137 # Dump the missing ones
 138 for kind in python_decl:
 139     for fullname in python_decl[kind]:
 140         print(" .. auto{}:: {}".format(kind, fullname))
 141
 142 ################ And now deal with Doxygen declarations
 143 ################
 144
 145 doxy_funs = {} # {classname: {func_name: [args]} }
 146 doxy_vars = {} # {classname: [names]}
 147
 148 # find the declarations in the XML files
 149 for arg in xml_files:
 150     if arg[-4:] != '.xml':
 151         print ("Argument '{}' does not end with '.xml'".format(arg))
 152         continue
 153     #print("Parse file {}".format(arg))
 154     tree = ET.parse(arg)
 155     for elem in tree.findall(".//compounddef"):
 156         if elem.attrib["kind"] == "class":
 157             if elem.attrib["prot"] != "public":
 158                 continue
 159             if "compoundname" in elem:
 160                 raise Exception("Compound {} has no 'compoundname' child tag.".format(elem))
 161             compoundname = elem.find("compoundname").text
 162             #print ("compoundname {}".format(compoundname))
 163         elif elem.attrib["kind"] == "file":
 164             compoundname = ""
 165         elif elem.attrib["kind"] == "namespace":
 166             compoundname = elem.find("compoundname").text
 167         else:
 168             print("Element {} is of kind {}".format(elem.attrib["id"], elem.attrib["kind"]))
 169
 170         for member in elem.findall('.//memberdef'):
 171             if member.attrib["prot"] != "public":
 172                 continue
 173             kind = member.attrib["kind"]
 174             name = member.find("name").text
 175             #print("kind:{} compoundname:{} name:{}".format( kind,compoundname, name))
 176             if kind == "variable":
 177                 if compoundname not in doxy_vars:
 178                     doxy_vars[compoundname] = []
 179                 doxy_vars[compoundname].append(name)
 180             elif kind == "function":
 181                 args = member.find('argsstring').text
 182                 args = re.sub('\)[^)]*$', ')', args) # ignore what's after the parameters (eg, '=0' or ' const')
 183
 184                 if compoundname not in doxy_funs:
 185                     doxy_funs[compoundname] = {}
 186                 if name not in doxy_funs[compoundname]:
 187                     doxy_funs[compoundname][name] = []
 188                 doxy_funs[compoundname][name].append(args)
 189             elif kind == "friend":
 190                 pass # Ignore friendship
 191             else:
 192                 print ("member {}::{} is of kind {}".format(compoundname, name, kind))
 193
 194 # Forget about the declarations that are done in the RST
 195 with os.popen('grep doxygenfunction:: find-missing.ignore source/*rst|sed \'s/^.*doxygenfunction:: //\'') as pse:
 196     for line in (l.strip() for l in pse):
 197         (klass, obj, args) = (None, None, None)
 198         if "(" in line:
 199             (line, args) = line.split('(', 1)
 200             args = "({}".format(args)
 201         if '::' in line:
 202             (klass, obj) = line.rsplit('::', 1)
 203         else:
 204             (klass, obj) = ("", line)
 205
 206         if klass not in doxy_funs:
 207             print("Warning: {} documented, but class {} not found in doxygen.".format(line, klass))
 208             continue
 209         if obj not in doxy_funs[klass]:
 210             print("Warning: Object '{}' documented but not found in '{}'".format(line, klass))
 211 #            for obj in doxy_funs[klass]:
 212 #                print("  found: {}::{}".format(klass, obj))
 213         elif len(doxy_funs[klass][obj])==1:
 214             del doxy_funs[klass][obj]
 215         elif args not in doxy_funs[klass][obj]:
 216             print("Warning: Function {}{} not found in {}".format(obj, args, klass))
 217         else:
 218             #print("Found {} in {}".format(line, klass))
 219             doxy_funs[klass][obj].remove(args)
 220             if len(doxy_funs[klass][obj]) == 0:
 221                 del doxy_funs[klass][obj]
 222 with os.popen('grep doxygenvariable:: find-missing.ignore source/*rst|sed \'s/^.*doxygenvariable:: //\'') as pse:
 223     for line in (l.strip() for l in pse):
 224         (klass, var) = line.rsplit('::', 1)
 225
 226         if klass not in doxy_vars:
 227             print("Warning: {} documented, but class {} not found in doxygen.".format(line, klass))
 228             continue
 229         if var not in doxy_vars[klass]:
 230             print("Warning: Object {} documented but not found in {}".format(line, klass))
 231         else:
 232 #            print("Found {} in {}".format(line, klass))
 233             doxy_vars[klass].remove(var)
 234             if len(doxy_vars[klass]) == 0:
 235                 del doxy_vars[klass]
 236
 237 # Dump the undocumented Doxygen declarations
 238 for obj in sorted(doxy_funs):
 239     for meth in sorted(doxy_funs[obj]):
 240         for args in sorted(doxy_funs[obj][meth]):
 241             print(".. doxygenfunction:: {}::{}{}".format(obj, meth, args))
 242
 243 for obj in doxy_vars:
 244     for meth in sorted(doxy_vars[obj]):
 245         print(".. doxygenvariable:: {}::{}".format(obj, meth))