2 # Copyright 2012-2015 Bronto Software, Inc. and contributors
4 # Licensed under the Apache License, Version 2.0 (the "License");
5 # you may not use this file except in compliance with the License.
6 # You may obtain a copy of the License at
8 # http://www.apache.org/licenses/LICENSE-2.0
10 # Unless required by applicable law or agreed to in writing, software
11 # distributed under the License is distributed on an "AS IS" BASIS,
12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 # See the License for the specific language governing permissions and
14 # limitations under the License.
17 from __future__ import print_function, unicode_literals
20 import cPickle as pickle
30 from optparse import OptionParser
34 import javasphinx.compiler as compiler
35 import javasphinx.util as util
38 if isinstance(s, str):
41 return s.encode('utf-8')
43 def find_source_files(input_path, excludes):
44 """ Get a list of filenames for all Java source files within the given
51 input_path = os.path.normpath(os.path.abspath(input_path))
53 for dirpath, dirnames, filenames in os.walk(input_path):
54 if is_excluded(dirpath, excludes):
58 for filename in filenames:
59 if filename.endswith(".java"):
60 java_files.append(os.path.join(dirpath, filename))
64 def write_toc(packages, opts):
66 doc.add_heading(opts.toc_title, '=')
68 toc = util.Directive('toctree')
69 toc.add_option('maxdepth', '2')
72 for package in sorted(packages.keys()):
73 toc.add_content("%s/package-index\n" % package.replace('.', '/'))
75 filename = 'packages.' + opts.suffix
76 fullpath = os.path.join(opts.destdir, filename)
78 if os.path.exists(fullpath) and not (opts.force or opts.update):
79 sys.stderr.write(fullpath + ' already exists. Use -f to overwrite.\n')
82 f = open(fullpath, 'w')
83 f.write(encode_output(doc.build()))
86 def write_documents(packages, documents, sources, opts):
87 package_contents = dict()
89 # Write individual documents
90 for fullname, (package, name, document) in documents.items():
91 if is_package_info_doc(name):
94 package_path = package.replace('.', os.sep)
95 filebasename = name.replace('.', '-')
96 filename = filebasename + '.' + opts.suffix
97 dirpath = os.path.join(opts.destdir, package_path)
98 fullpath = os.path.join(dirpath, filename)
100 if not os.path.exists(dirpath):
102 elif os.path.exists(fullpath) and not (opts.force or opts.update):
103 sys.stderr.write(fullpath + ' already exists. Use -f to overwrite.\n')
106 # Add to package indexes
107 package_contents.setdefault(package, list()).append(filebasename)
109 if opts.update and os.path.exists(fullpath):
110 # If the destination file is newer than the source file than skip
112 source_mod_time = os.stat(sources[fullname]).st_mtime
113 dest_mod_time = os.stat(fullpath).st_mtime
115 if source_mod_time < dest_mod_time:
118 f = open(fullpath, 'w')
119 f.write(encode_output(document))
122 # Write package-index for each package
123 for package, classes in package_contents.items():
124 doc = util.Document()
125 doc.add_heading(package, '=')
127 #Adds the package documentation (if any)
128 if packages[package] != '':
129 documentation = packages[package]
130 doc.add_line("\n%s" % documentation)
132 doc.add_object(util.Directive('java:package', package))
134 toc = util.Directive('toctree')
135 toc.add_option('maxdepth', '1')
138 for filebasename in classes:
139 toc.add_content(filebasename + '\n')
142 package_path = package.replace('.', os.sep)
143 filename = 'package-index.' + opts.suffix
144 dirpath = os.path.join(opts.destdir, package_path)
145 fullpath = os.path.join(dirpath, filename)
147 if not os.path.exists(dirpath):
149 elif os.path.exists(fullpath) and not (opts.force or opts.update):
150 sys.stderr.write(fullpath + ' already exists. Use -f to overwrite.\n')
153 f = open(fullpath, 'w')
154 f.write(encode_output(doc.build()))
158 if not os.path.exists(a):
161 if not os.path.exists(b):
164 a_mtime = int(os.stat(a).st_mtime)
165 b_mtime = int(os.stat(b).st_mtime)
167 if a_mtime < b_mtime:
172 def format_syntax_error(e):
177 rest = ' at %s line %d, character %d' % (value, pos[0], pos[1])
178 return e.description + rest
180 def generate_from_source_file(doc_compiler, source_file, cache_dir):
182 fingerprint = hashlib.md5(source_file.encode()).hexdigest()
183 cache_file = os.path.join(cache_dir, 'parsed-' + fingerprint + '.p')
185 if get_newer(source_file, cache_file) == cache_file:
186 return pickle.load(open(cache_file, 'rb'))
190 f = open(source_file)
195 ast = javalang.parse.parse(source)
196 except javalang.parser.JavaSyntaxError as e:
197 util.error('Syntax error in %s: %s', source_file, format_syntax_error(e))
199 util.unexpected('Unexpected exception while parsing %s', source_file)
203 if source_file.endswith("package-info.java"):
204 if ast.package is not None:
205 documentation = doc_compiler.compile_docblock(ast.package)
206 documents[ast.package.name] = (ast.package.name, 'package-info', documentation)
208 documents = doc_compiler.compile(ast)
210 util.unexpected('Unexpected exception while compiling %s', source_file)
213 dump_file = open(cache_file, 'wb')
214 pickle.dump(documents, dump_file)
219 def generate_documents(source_files, cache_dir, verbose, member_headers, parser):
222 doc_compiler = compiler.JavadocRestCompiler(None, member_headers, parser)
224 for source_file in source_files:
226 print('Processing', source_file)
228 this_file_documents = generate_from_source_file(doc_compiler, source_file, cache_dir)
229 for fullname in this_file_documents:
230 sources[fullname] = source_file
232 documents.update(this_file_documents)
234 #Existing packages dict, where each key is a package name
235 #and each value is the package documentation (if any)
238 #Gets the name of the package where the document was declared
239 #and adds it to the packages dict with no documentation.
240 #Package documentation, if any, will be collected from package-info.java files.
241 for package, name, _ in documents.values():
242 packages[package] = ""
244 #Gets packages documentation from package-info.java documents (if any).
245 for package, name, content in documents.values():
246 if is_package_info_doc(name):
247 packages[package] = content
249 return packages, documents, sources
251 def normalize_excludes(rootpath, excludes):
253 for exclude in excludes:
254 if not os.path.isabs(exclude) and not exclude.startswith(rootpath):
255 exclude = os.path.join(rootpath, exclude)
256 f_excludes.append(os.path.normpath(exclude) + os.path.sep)
259 def is_excluded(root, excludes):
261 if not root.endswith(sep):
263 for exclude in excludes:
264 if root.startswith(exclude):
268 def is_package_info_doc(document_name):
269 ''' Checks if the name of a document represents a package-info.java file. '''
270 return document_name == 'package-info'
273 def main(argv=sys.argv):
274 logging.basicConfig(level=logging.WARN)
276 parser = OptionParser(
278 usage: %prog [options] -o <output_path> <input_path> [exclude_paths, ...]
280 Look recursively in <input_path> for Java sources files and create reST files
281 for all non-private classes, organized by package under <output_path>. A package
282 index (package-index.<ext>) will be created for each package, and a top level
283 table of contents will be generated named packages.<ext>.
285 Paths matching any of the given exclude_paths (interpreted as regular
286 expressions) will be skipped.
288 Note: By default this script will not overwrite already created files.""")
290 parser.add_option('-o', '--output-dir', action='store', dest='destdir',
291 help='Directory to place all output', default='')
292 parser.add_option('-f', '--force', action='store_true', dest='force',
293 help='Overwrite all files')
294 parser.add_option('-c', '--cache-dir', action='store', dest='cache_dir',
295 help='Directory to stored cachable output')
296 parser.add_option('-u', '--update', action='store_true', dest='update',
297 help='Overwrite new and changed files', default=False)
298 parser.add_option('-T', '--no-toc', action='store_true', dest='notoc',
299 help='Don\'t create a table of contents file')
300 parser.add_option('-t', '--title', dest='toc_title', default='Javadoc',
301 help='Title to use on table of contents')
302 parser.add_option('--no-member-headers', action='store_false', default=True, dest='member_headers',
303 help='Don\'t generate headers for class members')
304 parser.add_option('-s', '--suffix', action='store', dest='suffix',
305 help='file suffix (default: rst)', default='rst')
306 parser.add_option('-I', '--include', action='append', dest='includes',
307 help='Additional input paths to scan', default=[])
308 parser.add_option('-p', '--parser', dest='parser_lib', default='lxml',
309 help='Beautiful Soup---html parser library option.')
310 parser.add_option('-v', '--verbose', action='store_true', dest='verbose',
311 help='verbose output')
313 (opts, args) = parser.parse_args(argv[1:])
316 parser.error('A source path is required.')
318 rootpath, excludes = args[0], args[1:]
320 input_paths = opts.includes
321 input_paths.append(rootpath)
324 parser.error('An output directory is required.')
326 if opts.suffix.startswith('.'):
327 opts.suffix = opts.suffix[1:]
329 for input_path in input_paths:
330 if not os.path.isdir(input_path):
331 sys.stderr.write('%s is not a directory.\n' % (input_path,))
334 if not os.path.isdir(opts.destdir):
335 os.makedirs(opts.destdir)
337 if opts.cache_dir and not os.path.isdir(opts.cache_dir):
338 os.makedirs(opts.cache_dir)
340 excludes = normalize_excludes(rootpath, excludes)
343 for input_path in input_paths:
344 source_files.extend(find_source_files(input_path, excludes))
346 packages, documents, sources = generate_documents(source_files, opts.cache_dir, opts.verbose,
347 opts.member_headers, opts.parser_lib)
349 write_documents(packages, documents, sources, opts)
352 write_toc(packages, opts)