From 1b966519556d1141719b7829011cdde06b9a11d6 Mon Sep 17 00:00:00 2001 From: Martin Quinson Date: Sat, 29 Jun 2019 00:35:47 +0200 Subject: [PATCH] embeed the javasphinx module I don't feel very good embeeding a software that was deprecated by its sole author, but I don't see no alternative module, and we need it to compile the documentation. I would like to distribute the documentation as a Debian package (or maybe re-add it to the tarballs), and I don't see any other options to get it working. --- docs/Build.sh | 2 +- docs/requirements.txt | 1 - docs/source/_ext/javasphinx-apidoc | 11 + docs/source/_ext/javasphinx/__init__.py | 24 + docs/source/_ext/javasphinx/apidoc.py | 352 ++++++++++++++ docs/source/_ext/javasphinx/compiler.py | 345 +++++++++++++ docs/source/_ext/javasphinx/domain.py | 594 +++++++++++++++++++++++ docs/source/_ext/javasphinx/extdoc.py | 124 +++++ docs/source/_ext/javasphinx/formatter.py | 163 +++++++ docs/source/_ext/javasphinx/htmlrst.py | 419 ++++++++++++++++ docs/source/_ext/javasphinx/util.py | 119 +++++ docs/source/conf.py | 2 +- 12 files changed, 2153 insertions(+), 3 deletions(-) create mode 100755 docs/source/_ext/javasphinx-apidoc create mode 100644 docs/source/_ext/javasphinx/__init__.py create mode 100644 docs/source/_ext/javasphinx/apidoc.py create mode 100644 docs/source/_ext/javasphinx/compiler.py create mode 100644 docs/source/_ext/javasphinx/domain.py create mode 100644 docs/source/_ext/javasphinx/extdoc.py create mode 100644 docs/source/_ext/javasphinx/formatter.py create mode 100644 docs/source/_ext/javasphinx/htmlrst.py create mode 100644 docs/source/_ext/javasphinx/util.py diff --git a/docs/Build.sh b/docs/Build.sh index 12fa8ceed2..1815b8b243 100755 --- a/docs/Build.sh +++ b/docs/Build.sh @@ -18,7 +18,7 @@ if [ "x$1" != 'xjava' -a -e source/java ] ; then echo "javasphinx not rerun: 'java' was not provided as an argument" else rm -rf source/java - javasphinx-apidoc --force -o source/java/ ../src/bindings/java/org/simgrid/msg + source/_ext/javasphinx-apidoc --force -o source/java/ ../src/bindings/java/org/simgrid/msg rm -f source/java/packages.rst # api_generated/source_java_packages.rst rm -f source/java/org/simgrid/msg/package-index.rst # api_generated/source_java_org_simgrid_msg_package-index.rst for f in source/java/org/simgrid/msg/* ; do diff --git a/docs/requirements.txt b/docs/requirements.txt index 2c64af2bfb..0a8f75d38f 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,5 +1,4 @@ breathe -javasphinx sphinx>=1.8.0 sphinx_rtd_theme diff --git a/docs/source/_ext/javasphinx-apidoc b/docs/source/_ext/javasphinx-apidoc new file mode 100755 index 0000000000..8de19768e8 --- /dev/null +++ b/docs/source/_ext/javasphinx-apidoc @@ -0,0 +1,11 @@ +#!/usr/bin/python3 + +# -*- coding: utf-8 -*- +import re +import sys + +from javasphinx.apidoc import main + +if __name__ == '__main__': + sys.argv[0] = re.sub(r'(-script\.pyw?|\.exe)?$', '', sys.argv[0]) + sys.exit(main()) diff --git a/docs/source/_ext/javasphinx/__init__.py b/docs/source/_ext/javasphinx/__init__.py new file mode 100644 index 0000000000..c6b9cb9bf0 --- /dev/null +++ b/docs/source/_ext/javasphinx/__init__.py @@ -0,0 +1,24 @@ +# +# Copyright 2012-2015 Bronto Software, Inc. and contributors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from .domain import JavaDomain +from .extdoc import javadoc_role + +def setup(app): + app.add_domain(JavaDomain) + + app.add_config_value('javadoc_url_map', dict(), '') + app.add_role('java:extdoc', javadoc_role) diff --git a/docs/source/_ext/javasphinx/apidoc.py b/docs/source/_ext/javasphinx/apidoc.py new file mode 100644 index 0000000000..b0de46f567 --- /dev/null +++ b/docs/source/_ext/javasphinx/apidoc.py @@ -0,0 +1,352 @@ +# +# Copyright 2012-2015 Bronto Software, Inc. and contributors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from __future__ import print_function, unicode_literals + +try: + import cPickle as pickle +except: + import pickle + +import hashlib +import logging +import sys +import os +import os.path + +from optparse import OptionParser + +import javalang + +import javasphinx.compiler as compiler +import javasphinx.util as util + +def encode_output(s): + if isinstance(s, str): + return s + else: + return s.encode('utf-8') + +def find_source_files(input_path, excludes): + """ Get a list of filenames for all Java source files within the given + directory. + + """ + + java_files = [] + + input_path = os.path.normpath(os.path.abspath(input_path)) + + for dirpath, dirnames, filenames in os.walk(input_path): + if is_excluded(dirpath, excludes): + del dirnames[:] + continue + + for filename in filenames: + if filename.endswith(".java"): + java_files.append(os.path.join(dirpath, filename)) + + return java_files + +def write_toc(packages, opts): + doc = util.Document() + doc.add_heading(opts.toc_title, '=') + + toc = util.Directive('toctree') + toc.add_option('maxdepth', '2') + doc.add_object(toc) + + for package in sorted(packages.keys()): + toc.add_content("%s/package-index\n" % package.replace('.', '/')) + + filename = 'packages.' + opts.suffix + fullpath = os.path.join(opts.destdir, filename) + + if os.path.exists(fullpath) and not (opts.force or opts.update): + sys.stderr.write(fullpath + ' already exists. Use -f to overwrite.\n') + sys.exit(1) + + f = open(fullpath, 'w') + f.write(encode_output(doc.build())) + f.close() + +def write_documents(packages, documents, sources, opts): + package_contents = dict() + + # Write individual documents + for fullname, (package, name, document) in documents.items(): + if is_package_info_doc(name): + continue + + package_path = package.replace('.', os.sep) + filebasename = name.replace('.', '-') + filename = filebasename + '.' + opts.suffix + dirpath = os.path.join(opts.destdir, package_path) + fullpath = os.path.join(dirpath, filename) + + if not os.path.exists(dirpath): + os.makedirs(dirpath) + elif os.path.exists(fullpath) and not (opts.force or opts.update): + sys.stderr.write(fullpath + ' already exists. Use -f to overwrite.\n') + sys.exit(1) + + # Add to package indexes + package_contents.setdefault(package, list()).append(filebasename) + + if opts.update and os.path.exists(fullpath): + # If the destination file is newer than the source file than skip + # writing it out + source_mod_time = os.stat(sources[fullname]).st_mtime + dest_mod_time = os.stat(fullpath).st_mtime + + if source_mod_time < dest_mod_time: + continue + + f = open(fullpath, 'w') + f.write(encode_output(document)) + f.close() + + # Write package-index for each package + for package, classes in package_contents.items(): + doc = util.Document() + doc.add_heading(package, '=') + + #Adds the package documentation (if any) + if packages[package] != '': + documentation = packages[package] + doc.add_line("\n%s" % documentation) + + doc.add_object(util.Directive('java:package', package)) + + toc = util.Directive('toctree') + toc.add_option('maxdepth', '1') + + classes.sort() + for filebasename in classes: + toc.add_content(filebasename + '\n') + doc.add_object(toc) + + package_path = package.replace('.', os.sep) + filename = 'package-index.' + opts.suffix + dirpath = os.path.join(opts.destdir, package_path) + fullpath = os.path.join(dirpath, filename) + + if not os.path.exists(dirpath): + os.makedirs(dirpath) + elif os.path.exists(fullpath) and not (opts.force or opts.update): + sys.stderr.write(fullpath + ' already exists. Use -f to overwrite.\n') + sys.exit(1) + + f = open(fullpath, 'w') + f.write(encode_output(doc.build())) + f.close() + +def get_newer(a, b): + if not os.path.exists(a): + return b + + if not os.path.exists(b): + return a + + a_mtime = int(os.stat(a).st_mtime) + b_mtime = int(os.stat(b).st_mtime) + + if a_mtime < b_mtime: + return b + + return a + +def format_syntax_error(e): + rest = "" + if e.at.position: + value = e.at.value + pos = e.at.position + rest = ' at %s line %d, character %d' % (value, pos[0], pos[1]) + return e.description + rest + +def generate_from_source_file(doc_compiler, source_file, cache_dir): + if cache_dir: + fingerprint = hashlib.md5(source_file.encode()).hexdigest() + cache_file = os.path.join(cache_dir, 'parsed-' + fingerprint + '.p') + + if get_newer(source_file, cache_file) == cache_file: + return pickle.load(open(cache_file, 'rb')) + else: + cache_file = None + + f = open(source_file) + source = f.read() + f.close() + + try: + ast = javalang.parse.parse(source) + except javalang.parser.JavaSyntaxError as e: + util.error('Syntax error in %s: %s', source_file, format_syntax_error(e)) + except Exception: + util.unexpected('Unexpected exception while parsing %s', source_file) + + documents = {} + try: + if source_file.endswith("package-info.java"): + if ast.package is not None: + documentation = doc_compiler.compile_docblock(ast.package) + documents[ast.package.name] = (ast.package.name, 'package-info', documentation) + else: + documents = doc_compiler.compile(ast) + except Exception: + util.unexpected('Unexpected exception while compiling %s', source_file) + + if cache_file: + dump_file = open(cache_file, 'wb') + pickle.dump(documents, dump_file) + dump_file.close() + + return documents + +def generate_documents(source_files, cache_dir, verbose, member_headers, parser): + documents = {} + sources = {} + doc_compiler = compiler.JavadocRestCompiler(None, member_headers, parser) + + for source_file in source_files: + if verbose: + print('Processing', source_file) + + this_file_documents = generate_from_source_file(doc_compiler, source_file, cache_dir) + for fullname in this_file_documents: + sources[fullname] = source_file + + documents.update(this_file_documents) + + #Existing packages dict, where each key is a package name + #and each value is the package documentation (if any) + packages = {} + + #Gets the name of the package where the document was declared + #and adds it to the packages dict with no documentation. + #Package documentation, if any, will be collected from package-info.java files. + for package, name, _ in documents.values(): + packages[package] = "" + + #Gets packages documentation from package-info.java documents (if any). + for package, name, content in documents.values(): + if is_package_info_doc(name): + packages[package] = content + + return packages, documents, sources + +def normalize_excludes(rootpath, excludes): + f_excludes = [] + for exclude in excludes: + if not os.path.isabs(exclude) and not exclude.startswith(rootpath): + exclude = os.path.join(rootpath, exclude) + f_excludes.append(os.path.normpath(exclude) + os.path.sep) + return f_excludes + +def is_excluded(root, excludes): + sep = os.path.sep + if not root.endswith(sep): + root += sep + for exclude in excludes: + if root.startswith(exclude): + return True + return False + +def is_package_info_doc(document_name): + ''' Checks if the name of a document represents a package-info.java file. ''' + return document_name == 'package-info' + + +def main(argv=sys.argv): + logging.basicConfig(level=logging.WARN) + + parser = OptionParser( + usage="""\ +usage: %prog [options] -o [exclude_paths, ...] + +Look recursively in for Java sources files and create reST files +for all non-private classes, organized by package under . A package +index (package-index.) will be created for each package, and a top level +table of contents will be generated named packages.. + +Paths matching any of the given exclude_paths (interpreted as regular +expressions) will be skipped. + +Note: By default this script will not overwrite already created files.""") + + parser.add_option('-o', '--output-dir', action='store', dest='destdir', + help='Directory to place all output', default='') + parser.add_option('-f', '--force', action='store_true', dest='force', + help='Overwrite all files') + parser.add_option('-c', '--cache-dir', action='store', dest='cache_dir', + help='Directory to stored cachable output') + parser.add_option('-u', '--update', action='store_true', dest='update', + help='Overwrite new and changed files', default=False) + parser.add_option('-T', '--no-toc', action='store_true', dest='notoc', + help='Don\'t create a table of contents file') + parser.add_option('-t', '--title', dest='toc_title', default='Javadoc', + help='Title to use on table of contents') + parser.add_option('--no-member-headers', action='store_false', default=True, dest='member_headers', + help='Don\'t generate headers for class members') + parser.add_option('-s', '--suffix', action='store', dest='suffix', + help='file suffix (default: rst)', default='rst') + parser.add_option('-I', '--include', action='append', dest='includes', + help='Additional input paths to scan', default=[]) + parser.add_option('-p', '--parser', dest='parser_lib', default='lxml', + help='Beautiful Soup---html parser library option.') + parser.add_option('-v', '--verbose', action='store_true', dest='verbose', + help='verbose output') + + (opts, args) = parser.parse_args(argv[1:]) + + if not args: + parser.error('A source path is required.') + + rootpath, excludes = args[0], args[1:] + + input_paths = opts.includes + input_paths.append(rootpath) + + if not opts.destdir: + parser.error('An output directory is required.') + + if opts.suffix.startswith('.'): + opts.suffix = opts.suffix[1:] + + for input_path in input_paths: + if not os.path.isdir(input_path): + sys.stderr.write('%s is not a directory.\n' % (input_path,)) + sys.exit(1) + + if not os.path.isdir(opts.destdir): + os.makedirs(opts.destdir) + + if opts.cache_dir and not os.path.isdir(opts.cache_dir): + os.makedirs(opts.cache_dir) + + excludes = normalize_excludes(rootpath, excludes) + source_files = [] + + for input_path in input_paths: + source_files.extend(find_source_files(input_path, excludes)) + + packages, documents, sources = generate_documents(source_files, opts.cache_dir, opts.verbose, + opts.member_headers, opts.parser_lib) + + write_documents(packages, documents, sources, opts) + + if not opts.notoc: + write_toc(packages, opts) diff --git a/docs/source/_ext/javasphinx/compiler.py b/docs/source/_ext/javasphinx/compiler.py new file mode 100644 index 0000000000..807d027025 --- /dev/null +++ b/docs/source/_ext/javasphinx/compiler.py @@ -0,0 +1,345 @@ +# +# Copyright 2012-2015 Bronto Software, Inc. and contributors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import javalang + +import javasphinx.formatter as formatter +import javasphinx.util as util +import javasphinx.htmlrst as htmlrst + +class JavadocRestCompiler(object): + """ Javadoc to ReST compiler. Builds ReST documentation from a Java syntax + tree. """ + + def __init__(self, filter=None, member_headers=True, parser='lxml'): + if filter: + self.filter = filter + else: + self.filter = self.__default_filter + + self.converter = htmlrst.Converter(parser) + + self.member_headers = member_headers + + def __default_filter(self, node): + """Excludes private members and those tagged "@hide" / "@exclude" in their + docblocks. + + """ + + if not isinstance(node, javalang.tree.Declaration): + return False + + if 'private' in node.modifiers: + return False + + if isinstance(node, javalang.tree.Documented) and node.documentation: + doc = javalang.javadoc.parse(node.documentation) + if 'hide' in doc.tags or 'exclude' in doc.tags: + return False + + return True + + def __html_to_rst(self, s): + return self.converter.convert(s) + + def __output_doc(self, documented): + if not isinstance(documented, javalang.tree.Documented): + raise ValueError('node not documented') + + output = util.Document() + + if not documented.documentation: + return output + + doc = javalang.javadoc.parse(documented.documentation) + + if doc.description: + output.add(self.__html_to_rst(doc.description)) + output.clear() + + if doc.authors: + output.add_line(':author: %s' % (self.__html_to_rst(', '.join(doc.authors)),)) + + for name, value in doc.params: + output.add_line(':param %s: %s' % (name, self.__html_to_rst(value))) + + for exception in doc.throws: + description = doc.throws[exception] + output.add_line(':throws %s: %s' % (exception, self.__html_to_rst(description))) + + if doc.return_doc: + output.add_line(':return: %s' % (self.__html_to_rst(doc.return_doc),)) + + if doc.tags.get('see'): + output.clear() + + see_also = ', '.join(self.__output_see(see) for see in doc.tags['see']) + output.add_line('**See also:** %s' % (see_also,)) + + return output + + def __output_see(self, see): + """ Convert the argument to a @see tag to rest """ + + if see.startswith('... + return self.__html_to_rst(see) + elif '"' in see: + # Plain text + return see + else: + # Type reference (default) + return ':java:ref:`%s`' % (see.replace('#', '.').replace(' ', ''),) + + def compile_type(self, declaration): + signature = util.StringBuilder() + formatter.output_declaration(declaration, signature) + + doc = self.__output_doc(declaration) + + directive = util.Directive('java:type', signature.build()) + directive.add_content(doc) + + return directive + + def compile_enum_constant(self, enum, constant): + signature = util.StringBuilder() + + for annotation in constant.annotations: + formatter.output_annotation(annotation, signature) + + # All enum constants are public, static, and final + signature.append('public static final ') + signature.append(enum) + signature.append(' ') + signature.append(constant.name) + + doc = self.__output_doc(constant) + + directive = util.Directive('java:field', signature.build()) + directive.add_content(doc) + + return directive + + def compile_field(self, field): + signature = util.StringBuilder() + + for annotation in field.annotations: + formatter.output_annotation(annotation, signature) + + formatter.output_modifiers(field.modifiers, signature) + signature.append(' ') + + formatter.output_type(field.type, signature) + signature.append(' ') + signature.append(field.declarators[0].name) + + doc = self.__output_doc(field) + + directive = util.Directive('java:field', signature.build()) + directive.add_content(doc) + + return directive + + def compile_constructor(self, constructor): + signature = util.StringBuilder() + + for annotation in constructor.annotations: + formatter.output_annotation(annotation, signature) + + formatter.output_modifiers(constructor.modifiers, signature) + signature.append(' ') + + if constructor.type_parameters: + formatter.output_type_params(constructor.type_parameters, signature) + signature.append(' ') + + signature.append(constructor.name) + + signature.append('(') + formatter.output_list(formatter.output_formal_param, constructor.parameters, signature, ', ') + signature.append(')') + + if constructor.throws: + signature.append(' throws ') + formatter.output_list(formatter.output_exception, constructor.throws, signature, ', ') + + doc = self.__output_doc(constructor) + + directive = util.Directive('java:constructor', signature.build()) + directive.add_content(doc) + + return directive + + def compile_method(self, method): + signature = util.StringBuilder() + + for annotation in method.annotations: + formatter.output_annotation(annotation, signature) + + formatter.output_modifiers(method.modifiers, signature) + signature.append(' ') + + if method.type_parameters: + formatter.output_type_params(method.type_parameters, signature) + signature.append(' ') + + formatter.output_type(method.return_type, signature) + signature.append(' ') + + signature.append(method.name) + + signature.append('(') + formatter.output_list(formatter.output_formal_param, method.parameters, signature, ', ') + signature.append(')') + + if method.throws: + signature.append(' throws ') + formatter.output_list(formatter.output_exception, method.throws, signature, ', ') + + doc = self.__output_doc(method) + + directive = util.Directive('java:method', signature.build()) + directive.add_content(doc) + + return directive + + def compile_type_document(self, imports_block, package, name, declaration): + """ Compile a complete document, documenting a type and its members """ + + outer_type = name.rpartition('.')[0] + + document = util.Document() + document.add(imports_block) + document.add_heading(name, '=') + + method_summary = util.StringBuilder() + document.add_object(method_summary) + + package_dir = util.Directive('java:package', package) + package_dir.add_option('noindex') + document.add_object(package_dir) + + # Add type-level documentation + type_dir = self.compile_type(declaration) + if outer_type: + type_dir.add_option('outertype', outer_type) + document.add_object(type_dir) + + if isinstance(declaration, javalang.tree.EnumDeclaration): + enum_constants = list(declaration.body.constants) + enum_constants.sort(key=lambda c: c.name) + + document.add_heading('Enum Constants') + for enum_constant in enum_constants: + if self.member_headers: + document.add_heading(enum_constant.name, '^') + c = self.compile_enum_constant(name, enum_constant) + c.add_option('outertype', name) + document.add_object(c) + + fields = list(filter(self.filter, declaration.fields)) + if fields: + document.add_heading('Fields', '-') + fields.sort(key=lambda f: f.declarators[0].name) + for field in fields: + if self.member_headers: + document.add_heading(field.declarators[0].name, '^') + f = self.compile_field(field) + f.add_option('outertype', name) + document.add_object(f) + + constructors = list(filter(self.filter, declaration.constructors)) + if constructors: + document.add_heading('Constructors', '-') + constructors.sort(key=lambda c: c.name) + for constructor in constructors: + if self.member_headers: + document.add_heading(constructor.name, '^') + c = self.compile_constructor(constructor) + c.add_option('outertype', name) + document.add_object(c) + + methods = list(filter(self.filter, declaration.methods)) + if methods: + document.add_heading('Methods', '-') + methods.sort(key=lambda m: m.name) + for method in methods: + if self.member_headers: + document.add_heading(method.name, '^') + m = self.compile_method(method) + m.add_option('outertype', name) + document.add_object(m) + + return document + + def compile(self, ast): + """ Compile autodocs for the given Java syntax tree. Documents will be + returned documenting each separate type. """ + + documents = {} + + imports = util.StringBuilder() + for imp in ast.imports: + if imp.static or imp.wildcard: + continue + + package_parts = [] + cls_parts = [] + + for part in imp.path.split('.'): + if cls_parts or part[0].isupper(): + cls_parts.append(part) + else: + package_parts.append(part) + + + # If the import's final part wasn't capitalized, + # append it to the class parts anyway so sphinx doesn't complain. + if cls_parts == []: + cls_parts.append(package_parts.pop()) + + package = '.'.join(package_parts) + cls = '.'.join(cls_parts) + + imports.append(util.Directive('java:import', package + ' ' + cls).build()) + import_block = imports.build() + + if not ast.package: + raise ValueError('File must have package declaration') + + package = ast.package.name + type_declarations = [] + for path, node in ast.filter(javalang.tree.TypeDeclaration): + if not self.filter(node): + continue + + classes = [n.name for n in path if isinstance(n, javalang.tree.TypeDeclaration)] + classes.append(node.name) + + name = '.'.join(classes) + type_declarations.append((package, name, node)) + + for package, name, declaration in type_declarations: + full_name = package + '.' + name + document = self.compile_type_document(import_block, package, name, declaration) + documents[full_name] = (package, name, document.build()) + return documents + + def compile_docblock(self, documented): + ''' Compiles a single, standalone docblock. ''' + return self.__output_doc(documented).build() diff --git a/docs/source/_ext/javasphinx/domain.py b/docs/source/_ext/javasphinx/domain.py new file mode 100644 index 0000000000..6b137fc10a --- /dev/null +++ b/docs/source/_ext/javasphinx/domain.py @@ -0,0 +1,594 @@ +# +# Copyright 2012-2015 Bronto Software, Inc. and contributors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import re +import string + +from docutils import nodes +from docutils.parsers.rst import Directive, directives + +from sphinx import addnodes, version_info +from sphinx.roles import XRefRole +from sphinx.locale import l_, _ +from sphinx.domains import Domain, ObjType +from sphinx.directives import ObjectDescription +from sphinx.util.nodes import make_refnode +from sphinx.util.docfields import Field, TypedField, GroupedField + +import javalang + +import javasphinx.extdoc as extdoc +import javasphinx.formatter as formatter +import javasphinx.util as util + +# Classes in java.lang. These are available without an import. +java_dot_lang = set([ + 'AbstractMethodError', 'Appendable', 'ArithmeticException', + 'ArrayIndexOutOfBoundsException', 'ArrayStoreException', 'AssertionError', + 'AutoCloseable', 'Boolean', 'BootstrapMethodError', 'Byte', 'Character', + 'CharSequence', 'Class', 'ClassCastException', 'ClassCircularityError', + 'ClassFormatError', 'ClassLoader', 'ClassNotFoundException', 'ClassValue', + 'Cloneable', 'CloneNotSupportedException', 'Comparable', 'Compiler', + 'Deprecated', 'Double', 'Enum', 'EnumConstantNotPresentException', 'Error', + 'Exception', 'ExceptionInInitializerError', 'Float', 'IllegalAccessError', + 'IllegalAccessException', 'IllegalArgumentException', + 'IllegalMonitorStateException', 'IllegalStateException', + 'IllegalThreadStateException', 'IncompatibleClassChangeError', + 'IndexOutOfBoundsException', 'InheritableThreadLocal', 'InstantiationError', + 'InstantiationException', 'Integer', 'InternalError', 'InterruptedException', + 'Iterable', 'LinkageError', 'Long', 'Math', 'NegativeArraySizeException', + 'NoClassDefFoundError', 'NoSuchFieldError', 'NoSuchFieldException', + 'NoSuchMethodError', 'NoSuchMethodException', 'NullPointerException', 'Number', + 'NumberFormatException', 'Object', 'OutOfMemoryError', 'Override', 'Package', + 'Process', 'ProcessBuilder', 'Readable', 'ReflectiveOperationException', + 'Runnable', 'Runtime', 'RuntimeException', 'RuntimePermission', 'SafeVarargs', + 'SecurityException', 'SecurityManager', 'Short', 'StackOverflowError', + 'StackTraceElement', 'StrictMath', 'String', 'StringBuffer', 'StringBuilder', + 'StringIndexOutOfBoundsException', 'SuppressWarnings', 'System', 'Thread', + 'ThreadDeath', 'ThreadGroup', 'ThreadLocal', 'Throwable', + 'TypeNotPresentException', 'UnknownError', 'UnsatisfiedLinkError', + 'UnsupportedClassVersionError', 'UnsupportedOperationException', 'VerifyError', + 'VirtualMachineError', 'Void']) + +class JavaObject(ObjectDescription): + option_spec = { + 'noindex': directives.flag, + 'package': directives.unchanged, + 'outertype': directives.unchanged + } + + def _build_ref_node(self, target): + ref = addnodes.pending_xref('', refdomain='java', reftype='type', reftarget=target, modname=None, classname=None) + ref['java:outertype'] = self.get_type() + + package = self.env.temp_data.get('java:imports', dict()).get(target, None) + + if not package and target in java_dot_lang: + package = 'java.lang' + + if package: + ref['java:imported'] = True + ref['java:package'] = package + else: + ref['java:imported'] = False + ref['java:package'] = self.get_package() + + return ref + + def _build_type_node(self, typ): + if isinstance(typ, javalang.tree.ReferenceType): + if typ.dimensions: + dim = '[]' * len(typ.dimensions) + else: + dim = '' + + target = typ.name + parts = [] + + while typ: + ref_node = self._build_ref_node(target) + ref_node += nodes.Text(typ.name, typ.name) + parts.append(ref_node) + + if typ.arguments: + parts.append(nodes.Text('<', '<')) + + first = True + for type_arg in typ.arguments: + if first: + first = False + else: + parts.append(nodes.Text(', ', ', ')) + + if type_arg.pattern_type == '?': + parts.append(nodes.Text('?', '?')) + else: + if type_arg.pattern_type: + s = '? %s ' % (type_arg.pattern_type,) + parts.append(nodes.Text(s, s)) + parts.extend(self._build_type_node(type_arg.type)) + + parts.append(nodes.Text('>', '>')) + + typ = typ.sub_type + + if typ: + target = target + '.' + typ.name + parts.append(nodes.Text('.', '.')) + elif dim: + parts.append(nodes.Text(dim, dim)) + + return parts + else: + type_repr = formatter.output_type(typ).build() + return [nodes.Text(type_repr, type_repr)] + + def _build_type_node_list(self, types): + parts = self._build_type_node(types[0]) + for typ in types[1:]: + parts.append(nodes.Text(', ', ', ')) + parts.extend(self._build_type_node(typ)) + return parts + + def handle_signature(self, sig, signode): + handle_name = 'handle_%s_signature' % (self.objtype,) + handle = getattr(self, handle_name, None) + + if handle: + return handle(sig, signode) + else: + raise NotImplementedError + + def get_index_text(self, package, type, name): + raise NotImplementedError + + def get_package(self): + return self.options.get('package', self.env.temp_data.get('java:package')) + + def get_type(self): + return self.options.get('outertype', '.'.join(self.env.temp_data.get('java:outertype', []))) + + def add_target_and_index(self, name, sig, signode): + package = self.get_package() + type = self.get_type(); + + fullname = '.'.join(filter(None, (package, type, name))) + basename = fullname.partition('(')[0] + + # note target + if fullname not in self.state.document.ids: + signode['names'].append(fullname) + signode['ids'].append(fullname) + signode['first'] = (not self.names) + self.state.document.note_explicit_target(signode) + + objects = self.env.domaindata['java']['objects'] + if fullname in objects: + self.state_machine.reporter.warning( + 'duplicate object description of %s, ' % fullname + + 'other instance in ' + self.env.doc2path(objects[fullname][0]) + + ', use :noindex: for one of them', + line=self.lineno) + + objects[fullname] = (self.env.docname, self.objtype, basename) + + indextext = self.get_index_text(package, type, name) + if indextext: + self.indexnode['entries'].append(_create_indexnode(indextext, fullname)) + + def before_content(self): + self.set_type = False + + if self.objtype == 'type' and self.names: + self.set_type = True + self.env.temp_data.setdefault('java:outertype', list()).append(self.names[0]) + + def after_content(self): + if self.set_type: + self.env.temp_data['java:outertype'].pop() + +class JavaMethod(JavaObject): + doc_field_types = [ + TypedField('parameter', label=l_('Parameters'), + names=('param', 'parameter', 'arg', 'argument'), + typerolename='type', typenames=('type',)), + Field('returnvalue', label=l_('Returns'), has_arg=False, + names=('returns', 'return')), + GroupedField('throws', names=('throws',), label=l_('Throws'), rolename='type') + ] + + def handle_method_signature(self, sig, signode): + try: + member = javalang.parse.parse_member_signature(sig) + except javalang.parser.JavaSyntaxError: + raise self.error("syntax error in method signature") + + if not isinstance(member, javalang.tree.MethodDeclaration): + raise self.error("expected method declaration") + + mods = formatter.output_modifiers(member.modifiers).build() + signode += nodes.Text(mods + ' ', mods + ' ') + + if member.type_parameters: + type_params = formatter.output_type_params(member.type_parameters).build() + signode += nodes.Text(type_params, type_params) + signode += nodes.Text(' ', ' ') + + rnode = addnodes.desc_type('', '') + rnode += self._build_type_node(member.return_type) + + signode += rnode + signode += nodes.Text(' ', ' ') + signode += addnodes.desc_name(member.name, member.name) + + paramlist = addnodes.desc_parameterlist() + for parameter in member.parameters: + param = addnodes.desc_parameter('', '', noemph=True) + param += self._build_type_node(parameter.type) + + if parameter.varargs: + param += nodes.Text('...', '') + + param += nodes.emphasis(' ' + parameter.name, ' ' + parameter.name) + paramlist += param + signode += paramlist + + param_reprs = [formatter.output_type(param.type, with_generics=False).build() for param in member.parameters] + return member.name + '(' + ', '.join(param_reprs) + ')' + + def get_index_text(self, package, type, name): + return _('%s (Java method)' % (name,)) + +class JavaConstructor(JavaObject): + doc_field_types = [ + TypedField('parameter', label=l_('Parameters'), + names=('param', 'parameter', 'arg', 'argument'), + typerolename='type', typenames=('type',)), + GroupedField('throws', names=('throws',), label=l_('Throws')) + ] + + def handle_constructor_signature(self, sig, signode): + try: + member = javalang.parse.parse_constructor_signature(sig) + except javalang.parser.JavaSyntaxError: + raise self.error("syntax error in constructor signature") + + if not isinstance(member, javalang.tree.ConstructorDeclaration): + raise self.error("expected constructor declaration") + + mods = formatter.output_modifiers(member.modifiers).build() + signode += nodes.Text(mods + ' ', mods + ' ') + + signode += addnodes.desc_name(member.name, member.name) + + paramlist = addnodes.desc_parameterlist() + for parameter in member.parameters: + param = addnodes.desc_parameter('', '', noemph=True) + param += self._build_type_node(parameter.type) + + if parameter.varargs: + param += nodes.Text('...', '') + + param += nodes.emphasis(' ' + parameter.name, ' ' + parameter.name) + paramlist += param + signode += paramlist + + param_reprs = [formatter.output_type(param.type, with_generics=False).build() for param in member.parameters] + return '%s(%s)' % (member.name, ', '.join(param_reprs)) + + def get_index_text(self, package, type, name): + return _('%s (Java constructor)' % (name,)) + +class JavaType(JavaObject): + doc_field_types = [ + GroupedField('parameter', names=('param',), label=l_('Parameters')) + ] + + declaration_type = None + + def handle_type_signature(self, sig, signode): + try: + member = javalang.parse.parse_type_signature(sig) + except javalang.parser.JavaSyntaxError: + raise self.error("syntax error in field signature") + + if isinstance(member, javalang.tree.ClassDeclaration): + self.declaration_type = 'class' + elif isinstance(member, javalang.tree.InterfaceDeclaration): + self.declaration_type = 'interface' + elif isinstance(member, javalang.tree.EnumDeclaration): + self.declaration_type = 'enum' + elif isinstance(member, javalang.tree.AnnotationDeclaration): + self.declaration_type = 'annotation' + else: + raise self.error("expected type declaration") + + mods = formatter.output_modifiers(member.modifiers).build() + signode += nodes.Text(mods + ' ', mods + ' ') + + if self.declaration_type == 'class': + signode += nodes.Text('class ', 'class ') + elif self.declaration_type == 'interface': + signode += nodes.Text('interface ', 'interface ') + elif self.declaration_type == 'enum': + signode += nodes.Text('enum ', 'enum ') + elif self.declaration_type == 'annotation': + signode += nodes.Text('@interface ', '@interface ') + + signode += addnodes.desc_name(member.name, member.name) + + if self.declaration_type in ('class', 'interface') and member.type_parameters: + type_params = formatter.output_type_params(member.type_parameters).build() + signode += nodes.Text(type_params, type_params) + + if self.declaration_type == 'class': + if member.extends: + extends = ' extends ' + signode += nodes.Text(extends, extends) + signode += self._build_type_node(member.extends) + if member.implements: + implements = ' implements ' + signode += nodes.Text(implements, implements) + signode += self._build_type_node_list(member.implements) + elif self.declaration_type == 'interface': + if member.extends: + extends = ' extends ' + signode += nodes.Text(extends, extends) + signode += self._build_type_node_list(member.extends) + elif self.declaration_type == 'enum': + if member.implements: + implements = ' implements ' + signode += nodes.Text(implements, implements) + signode += self._build_type_node_list(member.implements) + + return member.name + + def get_index_text(self, package, type, name): + return _('%s (Java %s)' % (name, self.declaration_type)) + +class JavaField(JavaObject): + def handle_field_signature(self, sig, signode): + try: + member = javalang.parse.parse_member_signature(sig) + except javalang.parser.JavaSyntaxError: + raise self.error("syntax error in field signature") + + if not isinstance(member, javalang.tree.FieldDeclaration): + raise self.error("expected field declaration") + + mods = formatter.output_modifiers(member.modifiers).build() + signode += nodes.Text(mods + ' ', mods + ' ') + + tnode = addnodes.desc_type('', '') + tnode += self._build_type_node(member.type) + + signode += tnode + signode += nodes.Text(' ', ' ') + + if len(member.declarators) > 1: + self.error('only one field may be documented at a time') + + declarator = member.declarators[0] + signode += addnodes.desc_name(declarator.name, declarator.name) + + dim = '[]' * len(declarator.dimensions) + signode += nodes.Text(dim) + + if declarator.initializer and isinstance(declarator.initializer, javalang.tree.Literal): + signode += nodes.Text(' = ' + declarator.initializer.value) + + return declarator.name + + def get_index_text(self, package, type, name): + return _('%s (Java field)' % (name,)) + +class JavaPackage(Directive): + """ + Directive to mark description of a new package. + """ + + has_content = False + required_arguments = 1 + optional_arguments = 0 + final_argument_whitespace = False + option_spec = { + 'noindex': directives.flag, + } + + def run(self): + env = self.state.document.settings.env + package = self.arguments[0].strip() + noindex = 'noindex' in self.options + env.temp_data['java:package'] = package + env.domaindata['java']['objects'][package] = (env.docname, 'package', package) + ret = [] + + if not noindex: + targetnode = nodes.target('', '', ids=['package-' + package], ismod=True) + self.state.document.note_explicit_target(targetnode) + + # the platform and synopsis aren't printed; in fact, they are only + # used in the modindex currently + ret.append(targetnode) + + indextext = _('%s (package)') % (package,) + inode = addnodes.index(entries=[_create_indexnode(indextext, 'package-' + package)]) + ret.append(inode) + + return ret + +class JavaImport(Directive): + """ + This directive is just to tell Sphinx the source of a referenced type. + """ + + has_content = False + required_arguments = 2 + optional_arguments = 0 + final_argument_whitespace = False + option_spec = {} + + def run(self): + env = self.state.document.settings.env + package, typename = self.arguments + + env.temp_data.setdefault('java:imports', dict())[typename] = package + return [] + +class JavaXRefRole(XRefRole): + def process_link(self, env, refnode, has_explicit_title, title, target): + refnode['java:outertype'] = '.'.join(env.temp_data.get('java:outertype', list())) + + target = target.lstrip('~') + + # Strip a method component from the target + basetype = target + if '(' in basetype: + basetype = basetype.partition('(')[0] + if '.' in basetype: + basetype = basetype.rpartition('.')[0] + + package = env.temp_data.get('java:imports', dict()).get(basetype, None) + + if package: + refnode['java:imported'] = True + refnode['java:package'] = package + else: + refnode['java:imported'] = False + refnode['java:package'] = env.temp_data.get('java:package') + + if not has_explicit_title: + # if the first character is a tilde, don't display the module/class + # parts of the contents + if title[0:1] == '~': + title = title.partition('(')[0] + title = title[1:] + dot = title.rfind('.') + if dot != -1: + title = title[dot+1:] + + return title, target + +class JavaDomain(Domain): + """Java language domain.""" + name = 'java' + label = 'Java' + + object_types = { + 'package': ObjType(l_('package'), 'package', 'ref'), + 'type': ObjType(l_('type'), 'type', 'ref'), + 'field': ObjType(l_('field'), 'field', 'ref'), + 'constructor': ObjType(l_('constructor'), 'construct', 'ref'), + 'method': ObjType(l_('method'), 'meth', 'ref') + } + + directives = { + 'package': JavaPackage, + 'type': JavaType, + 'field': JavaField, + 'constructor': JavaConstructor, + 'method': JavaMethod, + 'import': JavaImport + } + + roles = { + 'package': JavaXRefRole(), + 'type': JavaXRefRole(), + 'field': JavaXRefRole(), + 'construct': JavaXRefRole(), + 'meth': JavaXRefRole(), + 'ref': JavaXRefRole(), + } + + initial_data = { + 'objects': {}, # fullname -> docname, objtype, basename + } + + def clear_doc(self, docname): + objects = dict(self.data['objects']) + + for fullname, (fn, _, _) in objects.items(): + if fn == docname: + del self.data['objects'][fullname] + + def resolve_xref(self, env, fromdocname, builder, typ, target, node, contnode): + objects = self.data['objects'] + package = node.get('java:package') + imported = node.get('java:imported') + type_context = node.get('java:outertype') + + # Partial function to make building the response easier + make_ref = lambda fullname: make_refnode(builder, fromdocname, objects[fullname][0], fullname, contnode, fullname) + + # Check for fully qualified references + if target in objects: + return make_ref(target) + + # Try with package name prefixed + if package: + fullname = package + '.' + target + if fullname in objects: + return make_ref(fullname) + + # Try with package and type prefixed + if package and type_context: + fullname = package + '.' + type_context + '.' + target + if fullname in objects: + return make_ref(fullname) + + # Try to find a matching suffix + suffix = '.' + target + basename_match = None + basename_suffix = suffix.partition('(')[0] + + for fullname, (_, _, basename) in objects.items(): + if fullname.endswith(suffix): + return make_ref(fullname) + elif basename.endswith(basename_suffix): + basename_match = fullname + + if basename_match: + return make_ref(basename_match) + + # Try creating an external documentation reference + ref = extdoc.get_javadoc_ref(self.env, target, target) + + if not ref and target in java_dot_lang: + fulltarget = 'java.lang.' + target + ref = extdoc.get_javadoc_ref(self.env, fulltarget, fulltarget) + + # If the target was imported try with the package prefixed + if not ref and imported: + fulltarget = package + '.' + target + ref = extdoc.get_javadoc_ref(self.env, fulltarget, fulltarget) + + if ref: + ref.append(contnode) + return ref + else: + return None + + def get_objects(self): + for refname, (docname, type, _) in self.data['objects'].items(): + yield (refname, refname, type, docname, refname, 1) + + +def _create_indexnode(indextext, fullname): + # See https://github.com/sphinx-doc/sphinx/issues/2673 + if version_info < (1, 4): + return ('single', indextext, fullname, '') + else: + return ('single', indextext, fullname, '', None) diff --git a/docs/source/_ext/javasphinx/extdoc.py b/docs/source/_ext/javasphinx/extdoc.py new file mode 100644 index 0000000000..1586bc21c0 --- /dev/null +++ b/docs/source/_ext/javasphinx/extdoc.py @@ -0,0 +1,124 @@ +# +# Copyright 2012-2015 Bronto Software, Inc. and contributors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import re + +from docutils import nodes, utils +from sphinx.util.nodes import split_explicit_title + +def get_javadoc_ref(app, rawtext, text): + javadoc_url_map = app.config.javadoc_url_map + + # Add default Java SE sources + if not javadoc_url_map.get("java"): + javadoc_url_map["java"] = ("http://docs.oracle.com/javase/8/docs/api", 'javadoc8') + if not javadoc_url_map.get("javax"): + javadoc_url_map["javax"] = ("http://docs.oracle.com/javase/8/docs/api", 'javadoc8') + if not javadoc_url_map.get("org.xml"): + javadoc_url_map["org.xml"] = ("http://docs.oracle.com/javase/8/docs/api", 'javadoc8') + if not javadoc_url_map.get("org.w3c"): + javadoc_url_map["org.w3c"] = ("http://docs.oracle.com/javase/8/docs/api", 'javadoc8') + + source = None + package = '' + method = None + + if '(' in text: + # If the javadoc contains a line like this: + # {@link #sort(List)} + # there is no package so the text.rindex will fail + try: + split_point = text.rindex('.', 0, text.index('(')) + method = text[split_point + 1:] + text = text[:split_point] + except ValueError: + pass + + for pkg, (baseurl, ext_type) in javadoc_url_map.items(): + if text.startswith(pkg + '.') and len(pkg) > len(package): + source = baseurl, ext_type + package = pkg + + if not source: + return None + + baseurl, ext_type = source + + package_parts = [] + cls_parts = [] + + for part in text.split('.'): + if cls_parts or part[0].isupper(): + cls_parts.append(part) + else: + package_parts.append(part) + + package = '.'.join(package_parts) + cls = '.'.join(cls_parts) + + if not baseurl.endswith('/'): + baseurl = baseurl + '/' + + if ext_type == 'javadoc': + if not cls: + cls = 'package-summary' + source = baseurl + package.replace('.', '/') + '/' + cls + '.html' + if method: + source = source + '#' + method + elif ext_type == 'javadoc8': + if not cls: + cls = 'package-summary' + source = baseurl + package.replace('.', '/') + '/' + cls + '.html' + if method: + source = source + '#' + re.sub(r'[()]', '-', method) + elif ext_type == 'sphinx': + if not cls: + cls = 'package-index' + source = baseurl + package.replace('.', '/') + '/' + cls.replace('.', '-') + '.html' + if method: + source = source + '#' + package + '.' + cls + '.' + method + else: + raise ValueError('invalid target specifier ' + ext_type) + + title = '.'.join(filter(None, (package, cls, method))) + node = nodes.reference(rawtext, '') + node['refuri'] = source + node['reftitle'] = title + + return node + +def javadoc_role(name, rawtext, text, lineno, inliner, options={}, content=[]): + """ Role for linking to external Javadoc """ + + has_explicit_title, title, target = split_explicit_title(text) + title = utils.unescape(title) + target = utils.unescape(target) + + if not has_explicit_title: + target = target.lstrip('~') + + if title[0] == '~': + title = title[1:].rpartition('.')[2] + + app = inliner.document.settings.env.app + ref = get_javadoc_ref(app, rawtext, target) + + if not ref: + raise ValueError("no Javadoc source found for %s in javadoc_url_map" % (target,)) + + ref.append(nodes.Text(title, title)) + + return [ref], [] diff --git a/docs/source/_ext/javasphinx/formatter.py b/docs/source/_ext/javasphinx/formatter.py new file mode 100644 index 0000000000..51b4ce719b --- /dev/null +++ b/docs/source/_ext/javasphinx/formatter.py @@ -0,0 +1,163 @@ +# +# Copyright 2012-2015 Bronto Software, Inc. and contributors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +""" +Convert Java syntax tree nodes to string representations. + +""" + +import javalang + +from .util import StringBuilder + +# The order for displaying modifiers +__modifiers_order = ('public', 'protected', 'private', 'static', 'abstract', 'final', + 'native', 'synchronized', 'transient', 'volatile', 'strictfp') + +def formatter(f): + def _f(node, output=None, **kwargs): + if output is None: + output = StringBuilder() + + f(node, output, **kwargs) + return output + return _f + +def output_list(f, items, output=None, sep=', '): + if items: + f(items[0], output) + for item in items[1:]: + output.append(sep) + f(item, output) + +@formatter +def output_annotation(annotation, output): + output.append('@') + output.append(annotation.name) + output.append(' ') + +@formatter +def output_type(type, output, with_generics=True): + if not type: + output.append('void') + return + + if type.dimensions: + dim = '[]' * len(type.dimensions) + else: + dim = '' + + if isinstance(type, javalang.tree.BasicType): + output.append(type.name) + else: + while type: + output.append(type.name) + + if with_generics: + output_type_args(type.arguments, output) + + type = type.sub_type + + if type: + output.append('.') + output.append(dim) + +@formatter +def output_exception(exception, output): + output.append(exception) + +@formatter +def output_type_arg(type_arg, output): + if type_arg.pattern_type == '?': + output.append('?') + else: + if type_arg.pattern_type: + output.append('? ') + output.append(type_arg.pattern_type) + output.append(' ') + + output_type(type_arg.type, output) + +@formatter +def output_type_args(type_args, output): + if type_args: + output.append('<') + output_list(output_type_arg, type_args, output, ', ') + output.append('>') + +@formatter +def output_type_param(type_param, output): + output.append(type_param.name) + + if type_param.extends: + output.append(' extends ') + output_list(output_type, type_param.extends, output, ' & ') + +@formatter +def output_type_params(type_params, output): + if type_params: + output.append('<') + output_list(output_type_param, type_params, output, ', ') + output.append('>') + +@formatter +def output_declaration(declaration, output): + for annotation in declaration.annotations: + output_annotation(annotation, output) + + output_modifiers(declaration.modifiers, output) + output.append(' ') + + if isinstance(declaration, javalang.tree.ClassDeclaration): + output.append('class ') + elif isinstance(declaration, javalang.tree.EnumDeclaration): + output.append('enum ') + elif isinstance(declaration, javalang.tree.InterfaceDeclaration): + output.append('interface ') + elif isinstance(declaration, javalang.tree.AnnotationDeclaration): + output.append('@interface ') + + output.append(declaration.name) + + if isinstance(declaration, (javalang.tree.ClassDeclaration, javalang.tree.InterfaceDeclaration)): + output_type_params(declaration.type_parameters, output) + + if isinstance(declaration, javalang.tree.ClassDeclaration) and declaration.extends: + output.append(' extends ') + output_type(declaration.extends, output) + + if isinstance(declaration, javalang.tree.InterfaceDeclaration) and declaration.extends: + output.append(' extends ') + output_list(output_type, declaration.extends, output, ', ') + + if isinstance(declaration, (javalang.tree.ClassDeclaration, javalang.tree.EnumDeclaration)) and declaration.implements: + output.append(' implements ') + output_list(output_type, declaration.implements, output, ', ') + +@formatter +def output_formal_param(param, output): + output_type(param.type, output) + + if param.varargs: + output.append('...') + + output.append(' ') + output.append(param.name) + +@formatter +def output_modifiers(modifiers, output): + ordered_modifiers = [mod for mod in __modifiers_order if mod in modifiers] + output_list(lambda mod, output: output.append(mod), ordered_modifiers, output, ' ') diff --git a/docs/source/_ext/javasphinx/htmlrst.py b/docs/source/_ext/javasphinx/htmlrst.py new file mode 100644 index 0000000000..b34f1f205d --- /dev/null +++ b/docs/source/_ext/javasphinx/htmlrst.py @@ -0,0 +1,419 @@ +# +# Copyright 2013-2015 Bronto Software, Inc. and contributors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from __future__ import unicode_literals +from builtins import str + +import collections +import re + +from xml.sax.saxutils import escape as html_escape +from bs4 import BeautifulSoup + +Cell = collections.namedtuple('Cell', ['type', 'rowspan', 'colspan', 'contents']) + +class Converter(object): + def __init__(self, parser): + self._unknown_tags = set() + self._clear = '\n\n..\n\n' + + # Regular expressions + self._preprocess_anchors = re.compile(r'') + self._post_process_empty_lines = re.compile(r'^\s+$', re.MULTILINE) + self._post_process_compress_lines = re.compile(r'\n{3,}') + self._whitespace_with_newline = re.compile(r'[\s\n]+') + self._whitespace = re.compile(r'\s+') + self._html_tag = re.compile(r'<.*?>') + + self._preprocess_entity = re.compile(r'&(nbsp|lt|gt|amp)([^;]|[\n])') + self._parser = parser + + # -------------------------------------------------------------------------- + # ---- reST Utility Methods ---- + + def _unicode(self, s): + if isinstance(s, unicode): + return s + else: + return unicode(s, 'utf8') + + def _separate(self, s): + return u'\n\n' + s + u'\n\n' + + def _escape_inline(self, s): + return '\\ ' + s + '\\ ' + + def _inline(self, tag, s): + # Seems fishy if our inline markup spans lines. We will instead just return + # the string as is + if '\n' in s: + return s + + s = s.strip() + + if not s: + return s + + return self._escape_inline(tag + s.strip() + tag) + + def _role(self, role, s, label=None): + if label: + return self._escape_inline(':%s:`%s <%s>`' % (role, label, s)) + else: + return self._escape_inline(':%s:`%s`' % (role, s)) + + def _directive(self, directive, body=None): + header = '\n\n.. %s::\n\n' % (directive,) + + if body: + return header + self._left_justify(body, 3) + '\n\n' + else: + return header + '\n' + + def _hyperlink(self, target, label): + return self._escape_inline('`%s <%s>`_' % (label, target)) + + def _listing(self, marker, items): + items = [self._left_justify(item, len(marker) + 1) for item in items] + items = [marker + item[len(marker):] for item in items] + return self._separate('..') + self._separate('\n'.join(items)) + + def _left_justify(self, s, indent=0): + lines = [l.rstrip() for l in s.split('\n')] + indents = [len(l) - len(l.lstrip()) for l in lines if l] + + if not indents: + return s + + shift = indent - min(indents) + + if shift < 0: + return '\n'.join(l[-shift:] for l in lines) + else: + prefix = ' ' * shift + return '\n'.join(prefix + l for l in lines) + + def _compress_whitespace(self, s, replace=' ', newlines=True): + if newlines: + return self._whitespace_with_newline.sub(replace, s) + else: + return self._whitespace.sub(replace, s) + + # -------------------------------------------------------------------------- + # ---- DOM Tree Processing ---- + + def _process_table_cells(self, table): + """ Compile all the table cells. + + Returns a list of rows. The rows may have different lengths because of + column spans. + + """ + + rows = [] + + for i, tr in enumerate(table.find_all('tr')): + row = [] + + for c in tr.contents: + cell_type = getattr(c, 'name', None) + + if cell_type not in ('td', 'th'): + continue + + rowspan = int(c.attrs.get('rowspan', 1)) + colspan = int(c.attrs.get('colspan', 1)) + contents = self._process_children(c).strip() + + if cell_type == 'th' and i > 0: + contents = self._inline('**', contents) + + row.append(Cell(cell_type, rowspan, colspan, contents)) + + rows.append(row) + + return rows + + def _process_table(self, node): + rows = self._process_table_cells(node) + + if not rows: + return '' + + table_num_columns = max(sum(c.colspan for c in row) for row in rows) + + normalized = [] + + for row in rows: + row_num_columns = sum(c.colspan for c in row) + + if row_num_columns < table_num_columns: + cell_type = row[-1].type if row else 'td' + row.append(Cell(cell_type, 1, table_num_columns - row_num_columns, '')) + + col_widths = [0] * table_num_columns + row_heights = [0] * len(rows) + + for i, row in enumerate(rows): + j = 0 + for cell in row: + current_w = sum(col_widths[j:j + cell.colspan]) + required_w = max(len(l) for l in cell.contents.split('\n')) + + if required_w > current_w: + additional = required_w - current_w + col_widths[j] += additional - (cell.colspan - 1) * (additional // cell.colspan) + for jj in range(j + 1, j + cell.colspan): + col_widths[jj] += (additional // cell.colspan) + + current_h = row_heights[i] + required_h = len(cell.contents.split('\n')) + + if required_h > current_h: + row_heights[i] = required_h + + j += cell.colspan + + row_sep = '+' + '+'.join('-' * (l + 2) for l in col_widths) + '+' + header_sep = '+' + '+'.join('=' * (l + 2) for l in col_widths) + '+' + lines = [row_sep] + + for i, row in enumerate(rows): + for y in range(0, row_heights[i]): + line = [] + j = 0 + for c in row: + w = sum(n + 3 for n in col_widths[j:j+c.colspan]) - 2 + h = row_heights[i] + + line.append('| ') + cell_lines = c.contents.split('\n') + content = cell_lines[y] if y < len(cell_lines) else '' + line.append(content.ljust(w)) + + j += c.colspan + + line.append('|') + lines.append(''.join(line)) + + if i == 0 and all(c.type == 'th' for c in row): + lines.append(header_sep) + else: + lines.append(row_sep) + + return self._separate('\n'.join(lines)) + + def _process_children(self, node): + parts = [] + is_newline = False + + for c in node.contents: + part = self._process(c) + + if is_newline: + part = part.lstrip() + + if part: + parts.append(part) + is_newline = part.endswith('\n') + + return ''.join(parts) + + def _process_text(self, node): + return ''.join(node.strings) + + def _process(self, node): + if isinstance(node, str): + return self._compress_whitespace(node) + + simple_tags = { + 'b' : lambda s: self._inline('**', s), + 'strong' : lambda s: self._inline('**', s), + 'i' : lambda s: self._inline('*', s), + 'em' : lambda s: self._inline('*', s), + 'tt' : lambda s: self._inline('``', s), + 'code' : lambda s: self._inline('``', s), + 'h1' : lambda s: self._inline('**', s), + 'h2' : lambda s: self._inline('**', s), + 'h3' : lambda s: self._inline('**', s), + 'h4' : lambda s: self._inline('**', s), + 'h5' : lambda s: self._inline('**', s), + 'h6' : lambda s: self._inline('**', s), + 'sub' : lambda s: self._role('sub', s), + 'sup' : lambda s: self._role('sup', s), + 'hr' : lambda s: self._separate('') # Transitions not allowed + } + + if node.name in simple_tags: + return simple_tags[node.name](self._process_text(node)) + + if node.name == 'p': + return self._separate(self._process_children(node).strip()) + + if node.name == 'pre': + return self._directive('parsed-literal', self._process_text(node)) + + if node.name == 'a': + if 'name' in node.attrs: + return self._separate('.. _' + node['name'] + ':') + elif 'href' in node.attrs: + target = node['href'] + label = self._compress_whitespace(self._process_text(node).strip('\n')) + + if target.startswith('#'): + return self._role('ref', target[1:], label) + elif target.startswith('@'): + return self._role('java:ref', target[1:], label) + else: + return self._hyperlink(target, label) + + if node.name == 'ul': + items = [self._process(n) for n in node.find_all('li', recursive=False)] + return self._listing('*', items) + + if node.name == 'ol': + items = [self._process(n) for n in node.find_all('li', recursive=False)] + return self._listing('#.', items) + + if node.name == 'li': + s = self._process_children(node) + s = s.strip() + + # If it's multiline clear the end to correcly support nested lists + if '\n' in s: + s = s + '\n\n' + + return s + + if node.name == 'table': + return self._process_table(node) + + self._unknown_tags.add(node.name) + + return self._process_children(node) + + # -------------------------------------------------------------------------- + # ---- HTML Preprocessing ---- + + def _preprocess_inline_javadoc_replace(self, tag, f, s): + parts = [] + + start = '{@' + tag + start_length = len(start) + + i = s.find(start) + j = 0 + + while i != -1: + parts.append(s[j:i]) + + # Find a closing bracket such that the brackets are balanced between + # them. This is necessary since code examples containing { and } are + # commonly wrapped in {@code ...} tags + + try: + j = s.find('}', i + start_length) + 1 + while s.count('{', i, j) != s.count('}', i, j): + j = s.index('}', j) + 1 + except ValueError: + raise ValueError('Unbalanced {} brackets in ' + tag + ' tag') + + parts.append(f(s[i + start_length:j - 1].strip())) + i = s.find(start, j) + + parts.append(s[j:]) + + return ''.join(parts) + + def _preprocess_replace_javadoc_link(self, s): + s = self._compress_whitespace(s) + + target = None + label = '' + + if ' ' not in s: + target = s + else: + i = s.find(' ') + + while s.count('(', 0, i) != s.count(')', 0, i): + i = s.find(' ', i + 1) + + if i == -1: + i = len(s) + break + + target = s[:i] + label = s[i:] + + if target[0] == '#': + target = target[1:] + + target = target.replace('#', '.').replace(' ', '').strip() + + # Strip HTML tags from the target + target = self._html_tag.sub('', target) + + label = label.strip() + + return '%s' % (target, label) + + def _preprocess_close_anchor_tags(self, s): + # Add closing tags to all anchors so they are better handled by the parser + return self._preprocess_anchors.sub(r'', s) + + def _preprocess_fix_entities(self, s): + return self._preprocess_entity.sub(r'&\1;\2', s) + + def _preprocess(self, s_html): + to_tag = lambda t: lambda m: '<%s>%s' % (t, html_escape(m), t) + s_html = self._preprocess_inline_javadoc_replace('code', to_tag('code'), s_html) + s_html = self._preprocess_inline_javadoc_replace('literal', to_tag('span'), s_html) + s_html = self._preprocess_inline_javadoc_replace('docRoot', lambda m: '', s_html) + s_html = self._preprocess_inline_javadoc_replace('linkplain', self._preprocess_replace_javadoc_link, s_html) + s_html = self._preprocess_inline_javadoc_replace('link', self._preprocess_replace_javadoc_link, s_html) + + # Make sure all anchor tags are closed + s_html = self._preprocess_close_anchor_tags(s_html) + + # Fix up some entitities without closing ; + s_html = self._preprocess_fix_entities(s_html) + + return s_html + + # -------------------------------------------------------------------------- + # ---- Conversion entry point ---- + + def convert(self, s_html): + if not isinstance(s_html, str): + s_html = str(s_html, 'utf8') + + s_html = self._preprocess(s_html) + + if not s_html.strip(): + return '' + + soup = BeautifulSoup(s_html, self._parser) + top = soup.html.body + + result = self._process_children(top) + + # Post processing + result = self._post_process_empty_lines.sub('', result) + result = self._post_process_compress_lines.sub('\n\n', result) + result = result.strip() + + return result diff --git a/docs/source/_ext/javasphinx/util.py b/docs/source/_ext/javasphinx/util.py new file mode 100644 index 0000000000..2de85d5499 --- /dev/null +++ b/docs/source/_ext/javasphinx/util.py @@ -0,0 +1,119 @@ +# +# Copyright 2012-2015 Bronto Software, Inc. and contributors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from __future__ import unicode_literals +from builtins import str + +import logging +import re +import sys + +class StringBuilder(list): + def build(self): + return str(self) + + def __str__(self): + return ''.join(self) + +class Directive(object): + + def __init__(self, type, argument=''): + self.type = type + self.argument = argument + + self.options = [] + self.content = [] + + def add_option(self, name, value=''): + self.options.append((name, value)) + + def add_content(self, o): + assert o is not None + self.content.append(o) + + def build(self): + doc = Document() + doc.add_line('.. %s:: %s' % (self.type, self.argument)) + + for name, value in self.options: + doc.add_line(' :%s: %s\n' % (name, value)) + + content = Document() + + for obj in self.content: + content.add_object(obj) + + doc.clear() + for line in content.build().splitlines(): + doc.add_line(' ' + line) + doc.clear() + + return doc.build() + +class Document(object): + remove_trailing_whitespace_re = re.compile('[ \t]+$', re.MULTILINE) + collapse_empty_lines_re = re.compile('\n' + '{3,}', re.DOTALL) + + def __init__(self): + self.content = [] + + def add_object(self, o): + assert o is not None + + self.content.append(o) + + def add(self, s): + self.add_object(s) + + def add_line(self, s): + self.add(s) + self.add('\n') + + def add_heading(self, s, t='-'): + self.add_line(s) + self.add_line(t * len(s)) + + def clear(self): + self.add('\n\n') + + def build(self): + output = StringBuilder() + + for obj in self.content: + if isinstance(obj, Directive): + output.append('\n\n') + output.append(obj.build()) + output.append('\n\n') + elif isinstance(obj, Document): + output.append(obj.build()) + else: + output.append(str(obj)) + + output.append('\n\n') + + output = str(output) + output = self.remove_trailing_whitespace_re.sub('', output) + output = self.collapse_empty_lines_re.sub('\n\n', output) + + return output + +def error(s, *args, **kwargs): + logging.error(s, *args, **kwargs) + sys.exit(1) + +def unexpected(s, *args, **kwargs): + logging.exception(s, *args, **kwargs) + sys.exit(1) diff --git a/docs/source/conf.py b/docs/source/conf.py index 45042f22a6..abd7012c72 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -25,7 +25,7 @@ read_the_docs_build = os.environ.get('READTHEDOCS', None) == 'True' if read_the_docs_build: subprocess.call('cd source; doxygen', shell=True) - subprocess.call('javasphinx-apidoc --force -o source/java/ ../src/bindings/java/org/simgrid/msg', shell=True) + subprocess.call('source/_ext/javasphinx-apidoc --force -o source/java/ ../src/bindings/java/org/simgrid/msg', shell=True) subprocess.call('rm source/java/packages.rst', shell=True) # -- Project information ----------------------------------------------------- -- 2.20.1