#!/usr/bin/env python # @HEADER # ************************************************************************ # # TriBITS: Tribal Build, Integrate, and Test System # Copyright 2013 Sandia Corporation # # Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation, # the U.S. Government retains certain rights in this software. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are # met: # # 1. Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # # 3. Neither the name of the Corporation nor the names of the # contributors may be used to endorse or promote products derived from # this software without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, # PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # # ************************************************************************ # @HEADER import sys import os import subprocess import re import pprint import glob pp = pprint.PrettyPrinter(indent=4) # # Script documentation # usageHelp = r"""extract_rst_cmake_doc.py [OPTIONS] This script implements a system to extract RST-formatted (reStructuredText) comment blocks out of *.cmake files and populate them in a set of template *.rst files. An example of usage would be:: $ cd tribits/doc/developers_guide $ extract_rst_cmake_doc.py \ --extract-from=../../package_arch/,../..//utils/ \ --rst-file-pairs=TribitsDetailedMacroFunctionDocsTemplate.rst:TribitsDetailedMacroFunctionDocs.rst The --extract-from argument will read every *.cmake file in the directory given if the directory is specified with '/' at the end. We use --file-extensions (default is '.cmake') to provide the set of file patterns. The --rst-file-pairs argument gives pairs of files Template.rst:.rst that will subsutute the placeholders for macros and function definitions from the file Template.rst and will produce the file .rst. The way it works is that the *.cmake files specified by --extract-from must contain comment blocks of the form: {{{ # # @MACRO: SOME_MACRO_NAME() # # [rst formatted macro doc] # MACRO(SOME_MACRO_NAME ... ... # # @FUNCTION: SOME_FUNC_NAME() # # [rst formatted func doc] # FUNCTION(SOME_FUNC_NAME ... ... }}} The rules for these comment blocks are: * The comment blocks '#' must begin on the first column. * The special comment block identifier '@MACRO: ()' must start on the 3rd column, i.e. '# @MACRO: ()'. Otherwise, the comment block will be ignored. * Text in the comment blocks must begin after the 2nd column. That is, '#somthing' is illegal and will result in an error. * The comment blocks must terminate at the MACRO or FUNCTION definition and the of the macro or function listed in the marker '@MACRO: ()' or '@MACRO: ()' must match what is in the MACRO( ...) or FUNCTION( ...) definition. * Whitespce is allowed in the block marker between '@MACRO' and ':' and after the definition 'MACRO(' and 'FUNTION('. However, the name must be on the same line. These RST comment blocks are then inserted into RST template files Template.rst producing output files .rst listed in the --rst-file-pairs argument. The format of these RST template files is: {{{ Detailed Macro and Function Documentation ========================================= @FUNCTION: SOME_FUNC_NAME2() - @MACRO: SOME_MACRO_NAME1() - }}} The line of the form '@FUNCTION: SOME_FUNC_NAME2() -' are flagged as replacement blocks that are substitted with the extracted RST blocks of the same name and type. For example, the above RST template file would be substituted to look like: {{{ Detailed Macro and Function Documentation ========================================= SOME_FUNC_NAME2() ----------------- [rst formatted func text] SOME_MACRO_NAME1() ------------------ [rst formatted macro text] }}} The format of these substitution lines is: @: where: * is either 'MACRO' or 'FUNCTION'. * is the function or macro name (including the parath). * is a single char used to build the RST section underline. The rules for this format are: * The '@' in '@ must being on the first column. Otherwise the substitution block is ignored. * must match the name of a previously read in RST documentation block. If it is not, then processing ends with an error. * The must be present and must be a single char, typically '-', '+', "=", etc. This should be picked to be consistent with the RST document this is being put into. * At least one space must separate the fields ' ' or an error will occur. See the unit tests in extract_rst_cmake_doc_UnitTest.py for examples of behavior w.r.t. different errors and formatting mistakes. """ # # Helper functions # # Strip off the "()" for the entity name def getBaseEntityName(entityName): parenthIdx = entityName.find("(") if parenthIdx > 0: return entityName[0:parenthIdx] return entityName # Get the line type and entity name for CMake code def getLineEntityTypeAndName(line, fileNameAndLinePrefix, currentRstDocBlockType, inRstDocBlockBaseName \ ): lineSplit = line.split("(") if len(lineSplit) == 1: raise Exception(fileNameAndLinePrefix+ \ "error: expecting "+currentRstDocBlockType+"("+inRstDocBlockBaseName+" ...)"+\ " on this line. RST comment block must terminate in the stated entity!") lineEntityType = lineSplit[0].strip() lineEntityName = lineSplit[1].strip().split(" ")[0].strip() if lineEntityName[-1] == ")": lineEntityName = lineEntityName[0:-1] return (lineEntityType.upper(), lineEntityName) # Format file name and line number def getFileNameLineNum(fileName, lineNum): return fileName+":"+str(lineNum) # Format file name and line number as prefix def getFileNameLineNumPrefix(fileName, lineNum): return getFileNameLineNum(fileName, lineNum)+": " # Extract a set of RST doc blocks from the given text block (read from a file). # def extractRstDocBlocksFromText(rawText, rstBlockTypes, fileName, traceExtraction=False \ ): rstDocBlocks = {} inCommentBlock = False inRstDocBlock = "" currentRstDocBlockType = "" currentRstBlockBody = "" line_i = 0 for line in rawText.splitlines(): #print("\nline = '" + line + "'") line_i += 1 fileNameAndLinePrefix = fileName+":"+str(line_i)+": " # Determine if this is a comment line if len(line) == 0: inCommentBlock = False elif line[0] == '#': inCommentBlock = True else: inCommentBlock = False justFoundBlockBeginning = False if inCommentBlock and not inRstDocBlock: # Look for beginning of expected RST doc block type for blockType in rstBlockTypes: blockBeginning = "# @"+blockType lineBlockBeginning = line[0:len(blockBeginning)] #print("blockBeginning = '" + blockBeginning + "'") #print("lineBlockBeginning = '" + lineBlockBeginning + "'") if lineBlockBeginning == blockBeginning: #print("Matches block beginning " + blockType + "!") currentRstDocBlockType = blockType splitOnColon = line.split(":") if len(splitOnColon) != 2: raise Exception(fileNameAndLinePrefix+ \ "error: '"+line+"' is missing the colon ':' separator!") inRstDocBlock = splitOnColon[1].strip() #print("inRstDocBlock = '" + inRstDocBlock + "'") #print("currentRstDocBlockType = '" + currentRstDocBlockType + "'") inRstDocBlockFileNameLineNum = getFileNameLineNum(fileName, line_i) if traceExtraction: print("Extracting '" + blockType + "' block '" + inRstDocBlock + "'" + " from " + getFileNameLineNumPrefix(fileName, line_i)) currentRstBlockBody = "" justFoundBlockBeginning = True break if justFoundBlockBeginning: # Just found the beginning of the RST block so move to next line! continue if inCommentBlock and inRstDocBlock: # Strip off the beginning "# " and add to body if len(line) > 1 and line[1] != " ": raise Exception( \ getFileNameLineNumPrefix(fileName, line_i)+ \ "error: Comment blocks must have at least one space after '#'"+\ " in line = '"+line+"'") rstBlockBodyLine = line[2:] #print("rstBlockBodyLine = '" + rstBlockBodyLine + "'") currentRstBlockBody += (rstBlockBodyLine + "\n") #print("currentRstBlockBody = {\n" + currentRstBlockBody + "}") if not inCommentBlock and inRstDocBlock: # This is the end of the RST block fileNameAndLinePrefix = getFileNameLineNumPrefix(fileName, line_i) inRstDocBlockBaseName = getBaseEntityName(inRstDocBlock) # First, verify that the comment block matches the entity (lineEntityType, lineEntityName) = getLineEntityTypeAndName( line, fileNameAndLinePrefix, currentRstDocBlockType, inRstDocBlockBaseName) if lineEntityType != currentRstDocBlockType: raise Exception(fileNameAndLinePrefix+ \ "error: expecting '"+currentRstDocBlockType+\ "' but got type '"+lineEntityType+"''") if lineEntityName != inRstDocBlockBaseName: raise Exception(fileNameAndLinePrefix+ \ "error: expecting "+lineEntityType+" "+inRstDocBlock+\ " but got wrong "+lineEntityType+" name '"+lineEntityName+"'") # ToDo: Assert that blockName is the same! # Next, story the RST block rstDocBlocks.update( { inRstDocBlock : { "type" : currentRstDocBlockType, "body" : currentRstBlockBody, "fileNameLineNum" : inRstDocBlockFileNameLineNum, } } ) # Finally, rest to look for the next RST block inRstDocBlock = "" currentRstDocBlockType = "" #print("rstDocBlocks:\n" + str(rstDocBlocks)) # ToDo: Check that this line starts with currentRstDocBlockType or # throw exception! # end for line return rstDocBlocks # # Get an RST block from dict or throw except if it does not exist # def getRstDocBlock(rstDocBlocks, rstBlockName, fileNameAndLinePrefix, line): rstDocBlock = rstDocBlocks.get(rstBlockName, "") if rstDocBlock == "": raise Exception(fileNameAndLinePrefix+"Error, block name"+\ " '"+rstBlockName+"' does not exist in rstDocBlocks for line = '"+line+"'") return rstDocBlock # # Get RST section separator of length lineLen # def getRstSectStr(sepChar, lineLen): return sepChar * lineLen def removeEmtpyElements(arrayIn): arrayOut = [] for ele in arrayIn: if ele != "": arrayOut.append(ele) return arrayOut # Replace references to RST blocks with sections # def replaceWithRstDocBlocksInText(textToReplace, rstBlockTypes, rstDocBlocks, fileName, \ traceReplacements=False, fileNamePathBaseDir="", includeFileNameLineNum=False \ ): replacedText = "" textToReplaceLines = textToReplace.splitlines() numTextToReplaceLines = len(textToReplaceLines) if not fileNamePathBaseDir == "": fileNamePathBaseDirAndSepLen = len(fileNamePathBaseDir) if not fileNamePathBaseDir.endswith("/"): fileNamePathBaseDirAndSepLen += 1 else: fileNamePathBaseDirAndSepLen = 0 line_i = 0 for line in textToReplaceLines: line_i += 1 #print("\nline_i = " + str(line_i)) #print("line = '" + line + "'") # Look for block of text to replace replacedRstBlock = False for blockType in rstBlockTypes: blockBeginning = "@"+blockType lineBlockBeginning = line[0:len(blockBeginning)] #print("blockBeginning = '" + blockBeginning + "'") #print("lineBlockBeginning = '" + lineBlockBeginning + "'") if lineBlockBeginning == blockBeginning: #print("Matches block beginning " + blockType + "!") lineSplitArray = line.split(":") fileNameAndLinePrefix = fileName+":"+str(line_i)+": " #print("lineSplitArray = " + str(lineSplitArray)) if len(lineSplitArray)==1: # Missing the colon so no replacement! break rstBlockNameAndSecCharArray = removeEmtpyElements( lineSplitArray[1].strip().split(" ")) #print("rstBlockNameAndSecCharArray = " + str(rstBlockNameAndSecCharArray)) if len(rstBlockNameAndSecCharArray) != 2: raise Exception(fileNameAndLinePrefix+"Error, line does not match format"+\ " '@"+blockType+": ' for line = '"+line+"'") rstBlockName = rstBlockNameAndSecCharArray[0].strip() rstBlockSecChar = rstBlockNameAndSecCharArray[1].strip() #print("rstBlockName = '" + rstBlockName + "'") #print("rstBlockSecChar = '" + rstBlockSecChar + "'") if len(rstBlockSecChar) != 1: raise Exception(fileNameAndLinePrefix+\ "Error, separation char must be on char in line = '"+line+"'") if traceReplacements: print("Replacing '" + blockType + "' block '" + rstBlockName + "'" + " in " + getFileNameLineNumPrefix(fileName, line_i)) rstDocBlock = getRstDocBlock(rstDocBlocks, rstBlockName, fileNameAndLinePrefix, line) if rstDocBlock.get("type") != blockType: raise Exception(fileNameAndLinePrefix+"Error, looked up block type"+\ " '@"+rstDocBlock.get("type")+"' does not match block type for line = '"+line+"'") rstSecStr = getRstSectStr(rstBlockSecChar, len(rstBlockName)) # Add the replaced block section replacedText += (rstBlockName+"\n") replacedText += (rstSecStr+"\n") replacedText += rstDocBlock.get("body") if includeFileNameLineNum: fileNameLineNum = rstDocBlock.get("fileNameLineNum") if fileNamePathBaseDirAndSepLen > 0: relFileNameLineNum = fileNameLineNum[fileNamePathBaseDirAndSepLen:] else: relFileNameLineNum = fileNameLineNum replacedText += "\nIn: "+relFileNameLineNum + "\n\n" replacedRstBlock = True break if not replacedRstBlock and not (line_i == numTextToReplaceLines and line==""): replacedText += (line+"\n") #print("replacedText = {\n" + replacedText + "}") # end for line return replacedText # Get a list of files give list of files, dirs, and extensions def getExtractFilesList(inOptions): filesList = [] for fileOrDir in inOptions.extractFrom.split(","): if fileOrDir[-1] == "/": # This is a directory so grab all files with the matching extensions for fileExtension in inOptions.fileExtensions.split(","): globStr = fileOrDir+"*"+fileExtension globedFiles = glob.glob(globStr) filesList.extend(globedFiles) else: filesList.append(fileOrDir) return filesList # Get the list of template and replacement files def getRstFilesList(inOptions): rstFilesList = [] if inOptions.rstFilePairs: for templateFileAndFile in inOptions.rstFilePairs.split(","): rstFilesList.append(templateFileAndFile.split(":")) return rstFilesList # Extract RST blocks out of a set of files def extractRstDocBlocksFromFileList(fileList, rstBlockTypes, traceExtraction=False): rstDocBlocks = {} for fileName in fileList: rawFileStr = open(fileName, 'r').read() rstDocBlocks.update( extractRstDocBlocksFromText(rawFileStr, rstBlockTypes, fileName, traceExtraction) ) return rstDocBlocks # Make substitutions in a template file and produce a target file def replaceWithRstDocBlocksInTemplateFileList(rstFilesList, rstBlockTypes, rstDocBlocks, traceReplacements=False, fileNamePathBaseDir="", includeFileNameLineNum=False \ ): for (templateFileName, fileName) in rstFilesList: templateFileStr = open(templateFileName, 'r').read() fileStr = replaceWithRstDocBlocksInText(templateFileStr, rstBlockTypes, rstDocBlocks, templateFileName, traceReplacements=traceReplacements, fileNamePathBaseDir=fileNamePathBaseDir, includeFileNameLineNum=includeFileNameLineNum ) open(fileName, 'w').write(fileStr) # Run a command and synchronize the output def runCmnd(options, cmnd): if options.debug: print("*** Running command: " + cmnd) if options.noOpt: print(cmnd) else: child = subprocess.Popen(cmnd, stdout=subprocess.PIPE).stdout output = child.read() sys.stdout.flush() print(output) sys.stdout.flush() # # Script body # if __name__ == '__main__': # # A) Get command-line options # from optparse import OptionParser clp = OptionParser(usage=usageHelp) clp.add_option( "--extract-from", dest="extractFrom", type="string", default="", help="List (comma separated) of directories (ending with '/') and files that the RST"+\ " comment blocks will be extracted from. Directories ending with '/' will result in"+\ " files being found with extensions listed in --file-extensions. Otherwise, the given"+\ " files are searched.") clp.add_option( "--file-extensions", dest="fileExtensions", type="string", default=".cmake", help="List (comma separated) of file extensions (e.g. '.cmake'). The default"+\ "is '.cmake'") clp.add_option( "--rst-file-pairs", dest="rstFilePairs", type="string", default="", help="List (comma separated) files of the form 'Template.rst:.rst,...'"+\ " that gives sets of template files to have comment blocks inserted into.") clp.add_option( "--show-file-name-line-num", dest="showFileNameLineNum", action="store_true", help="Include the relative file name path and line number for each documentation block.") clp.add_option( "--no-show-file-name-line-num", dest="showFileNameLineNum", action="store_false", default=False, help="Include the relative file name path and line number for each documentation block. [default]" ) clp.add_option( "--file-name-path-base-dir", dest="fileNamePathBaseDir", type="string", default="", help="Base path stripped off of file names reported when --show-file-name-line-num is set."+\ " NOTE: This path should be relative to the paths in --extract-from and may be relative paths." ) clp.add_option( "--dump-rst-blocks", dest="dumpRstBlocks", action="store_true", help="Print out the RST documentation blocks that are read in.") clp.add_option( "--no-dump-rst-blocks", dest="dumpRstBlocks", action="store_false", default=False, help="Print out the RST documentation blocks that are read in. [default]" ) clp.add_option( "--do-trace", dest="doTrace", action="store_true", help="Print out trace of RST blocks read and replaced.") clp.add_option( "--no-do-trace", dest="doTrace", action="store_false", default=False, help="Do not print out trace of RST blocks read and replaced. [default]" ) (options, args) = clp.parse_args() # Echo the command-line if options.doTrace: print("") print("**************************************************************************") print("Script: extract_rst_cmake_doc.py \\") print(" --extract-from='" + options.extractFrom + "' \\") print(" --file-extensions='" + options.fileExtensions + "' \\") print(" --rst-file-pairs='" + options.rstFilePairs + "' \\") if options.showFileNameLineNum: print(" --show-file-name-line-num \\") else: print(" --no-show-file-name-line-num \\") print(" --file-name-path-base-dir='" + options.fileNamePathBaseDir + "' \\") if options.dumpRstBlocks: print(" --dump-rst-blocks \\") else: print(" --no-dump-rst-blocks \\") if options.doTrace: print(" --do-trace \\") else: print(" --no-do-trace \\") # Assert the commandline if not options.extractFrom: print("\nError, --extract-from must be specified (see --help)!") sys.exit(1) if options.doTrace and not options.rstFilePairs: print("\nWarning: --rst-file-pairs is empty and no RST comment blocks will be set!") # # B) Read in all of the RST documentation blocks # rstBlockTypes = ["MACRO", "FUNCTION"] if options.doTrace: print("\nExtracting RST documentation blocks in --extract-from:") extractFromFilesList = getExtractFilesList(options) rstDocBlocks = extractRstDocBlocksFromFileList( extractFromFilesList, rstBlockTypes, options.doTrace) if options.dumpRstBlocks: print("Read in RST blocks:\n") pp.pprint(rstDocBlocks) # # C) Make the substitutions in all of the file pairs # if options.doTrace: print("\nReplacing RST documentation blocks in RST files in --rst-file-pairs:") rstFilesList = getRstFilesList(options) replaceWithRstDocBlocksInTemplateFileList(rstFilesList, rstBlockTypes, rstDocBlocks, traceReplacements=options.doTrace, fileNamePathBaseDir=options.fileNamePathBaseDir, includeFileNameLineNum=options.showFileNameLineNum )