430 lines
19 KiB
Python
430 lines
19 KiB
Python
|
#!/usr/bin/python
|
||
|
|
|
||
|
|
#
|
||
|
|
# Copyright (c) 2017-2020 Intel Corporation
|
||
|
|
#
|
||
|
|
# Helper script for code formatting using clang-format-3.9 and autopep
|
||
|
|
|
||
|
|
import argparse
|
||
|
|
import filecmp
|
||
|
|
import os
|
||
|
|
import re
|
||
|
|
import sets
|
||
|
|
import subprocess
|
||
|
|
import sys
|
||
|
|
from termcolor import cprint
|
||
|
|
|
||
|
|
SCRIPT_VERSION = "1.3"
|
||
|
|
|
||
|
|
|
||
|
|
class CodeFormatter:
|
||
|
|
|
||
|
|
def __init__(self, command, expectedVersion, formatCommandArguments, verifyCommandArguments, verifyOutputIsDiff, fileEndings, fileDescription, installCommand):
|
||
|
|
self.command = command
|
||
|
|
self.expectedVersion = expectedVersion
|
||
|
|
self.formatCommandArguments = formatCommandArguments
|
||
|
|
self.verifyCommandArguments = verifyCommandArguments
|
||
|
|
self.verifyOutputIsDiff = verifyOutputIsDiff
|
||
|
|
self.fileEndings = fileEndings
|
||
|
|
self.fileDescription = fileDescription
|
||
|
|
self.installCommand = installCommand
|
||
|
|
|
||
|
|
def verifyFormatterVersion(self):
|
||
|
|
try:
|
||
|
|
versionOutput = subprocess.check_output([self.command, "--version"]).rstrip('\r\n')
|
||
|
|
if self.expectedVersion != "":
|
||
|
|
if versionOutput.startswith(self.expectedVersion):
|
||
|
|
print("[OK] Found formatter '" + versionOutput + "'")
|
||
|
|
return
|
||
|
|
else:
|
||
|
|
cprint("[NOT OK] Found '" + versionOutput + "'", "red")
|
||
|
|
cprint("[NOT OK] Version string does not start with '" + self.expectedVersion + "'", "red")
|
||
|
|
else:
|
||
|
|
return
|
||
|
|
except:
|
||
|
|
cprint("[ERROR] Could not run " + self.command, "red")
|
||
|
|
cprint("[INFO] Please check if correct version is installed or install with '" +
|
||
|
|
self.installCommand + "'", "blue")
|
||
|
|
sys.exit(1)
|
||
|
|
|
||
|
|
def printInputFiles(self):
|
||
|
|
if len(self.inputFiles) > 0:
|
||
|
|
print("Found " + self.fileDescription + " files:")
|
||
|
|
for fileName in self.inputFiles:
|
||
|
|
print(fileName)
|
||
|
|
print("")
|
||
|
|
|
||
|
|
def formatFile(self, fileName):
|
||
|
|
commandList = [self.command]
|
||
|
|
commandList.extend(self.formatCommandArguments)
|
||
|
|
commandList.append(fileName)
|
||
|
|
try:
|
||
|
|
subprocess.check_output(commandList, stderr=subprocess.STDOUT)
|
||
|
|
print("[OK] " + fileName)
|
||
|
|
except subprocess.CalledProcessError as e:
|
||
|
|
cprint("[ERROR] " + fileName + " (" + e.output.rstrip('\r\n') + ")", "red")
|
||
|
|
return True
|
||
|
|
return False
|
||
|
|
|
||
|
|
def performGitDiff(self, fileName, verifyOutput):
|
||
|
|
try:
|
||
|
|
diffProcess = subprocess.Popen(
|
||
|
|
["git", "diff", "--color=always", "--exit-code", "--no-index", "--", fileName, "-"],
|
||
|
|
stdin=subprocess.PIPE,
|
||
|
|
stdout=subprocess.PIPE,
|
||
|
|
stderr=subprocess.PIPE)
|
||
|
|
diffOutput, _ = diffProcess.communicate(verifyOutput)
|
||
|
|
if diffProcess.returncode == 0:
|
||
|
|
diffOutput = ""
|
||
|
|
except OSError:
|
||
|
|
cprint("[ERROR] Failed to run git diff on " + fileName, "red")
|
||
|
|
return (True, "")
|
||
|
|
return (False, diffOutput)
|
||
|
|
|
||
|
|
def verifyFile(self, fileName, printDiff):
|
||
|
|
commandList = [self.command]
|
||
|
|
commandList.extend(self.verifyCommandArguments)
|
||
|
|
commandList.append(fileName)
|
||
|
|
try:
|
||
|
|
verifyOutput = subprocess.check_output(commandList, stderr=subprocess.STDOUT)
|
||
|
|
except subprocess.CalledProcessError as e:
|
||
|
|
cprint("[ERROR] " + fileName + " (" + e.output.rstrip('\r\n') + ")", "red")
|
||
|
|
return True
|
||
|
|
|
||
|
|
diffOutput = ""
|
||
|
|
if self.verifyOutputIsDiff:
|
||
|
|
diffOutput = verifyOutput
|
||
|
|
else:
|
||
|
|
status, diffOutput = self.performGitDiff(fileName, verifyOutput)
|
||
|
|
if status:
|
||
|
|
return True
|
||
|
|
|
||
|
|
if diffOutput != "":
|
||
|
|
cprint("[NOT OK] " + fileName, "red")
|
||
|
|
if printDiff:
|
||
|
|
print(diffOutput.rstrip('\r\n'))
|
||
|
|
return True
|
||
|
|
|
||
|
|
print("[OK] " + fileName)
|
||
|
|
return False
|
||
|
|
|
||
|
|
|
||
|
|
class CodeFormatterClang(CodeFormatter):
|
||
|
|
CLANG_FORMAT_FILE = ".clang-format"
|
||
|
|
CHECKED_IN_CLANG_FORMAT_FILE = "clang-format"
|
||
|
|
CODE_FORMAT_IGNORE_FILE = ".codeformatignore"
|
||
|
|
|
||
|
|
def __init__(self):
|
||
|
|
CodeFormatter.__init__(self,
|
||
|
|
command="clang-format-3.9",
|
||
|
|
expectedVersion="clang-format version 3.9",
|
||
|
|
formatCommandArguments=["-style=file", "-fallback-style=none", "-i"],
|
||
|
|
verifyCommandArguments=["-style=file", "-fallback-style=none"],
|
||
|
|
verifyOutputIsDiff=False,
|
||
|
|
fileEndings=["cpp", "hpp", "c", "h", "cc"],
|
||
|
|
fileDescription="source and header",
|
||
|
|
installCommand="sudo apt-get install clang-format-3.9")
|
||
|
|
self.scriptPath = os.path.dirname(os.path.abspath(__file__))
|
||
|
|
self.checkedInClangFormatFile = os.path.join(self.scriptPath, CodeFormatterClang.CHECKED_IN_CLANG_FORMAT_FILE)
|
||
|
|
|
||
|
|
def verifyFormatterVersion(self):
|
||
|
|
CodeFormatter.verifyFormatterVersion(self)
|
||
|
|
self.verifyClangFormatFileExistsAndMatchesCheckedIn()
|
||
|
|
|
||
|
|
def verifyCheckedInClangFormatFileExists(self):
|
||
|
|
if os.path.exists(self.checkedInClangFormatFile):
|
||
|
|
print("[OK] Found " + CodeFormatterClang.CHECKED_IN_CLANG_FORMAT_FILE + " file (the one that should be in a repository) " +
|
||
|
|
self.checkedInClangFormatFile)
|
||
|
|
else:
|
||
|
|
cprint("[WARN] Not found " + CodeFormatterClang.CHECKED_IN_CLANG_FORMAT_FILE + " file " +
|
||
|
|
self.checkedInClangFormatFile, "yellow")
|
||
|
|
self.confirmWithUserClangFormatFileCantBeVerified()
|
||
|
|
|
||
|
|
def confirmWithUserClangFormatFileCantBeVerified(self):
|
||
|
|
if not self.args.yes:
|
||
|
|
answer = raw_input("Are you sure your .clang-format file is up-to-date and you want to continue? (y/N)")
|
||
|
|
if answer != "y":
|
||
|
|
sys.exit(1)
|
||
|
|
|
||
|
|
def verifyClangFormatFileExistsAndMatchesCheckedIn(self):
|
||
|
|
self.verifyCheckedInClangFormatFileExists()
|
||
|
|
foundClangFormatFiles = sets.Set()
|
||
|
|
for fileName in self.inputFiles:
|
||
|
|
dirName = os.path.dirname(os.path.abspath(fileName))
|
||
|
|
if not self.findClangFormatFileStartingFrom(dirName, fileName, foundClangFormatFiles):
|
||
|
|
sys.exit(1)
|
||
|
|
|
||
|
|
def findClangFormatFileStartingFrom(self, dirName, fileName, foundClangFormatFiles):
|
||
|
|
clangFormatFile = os.path.join(dirName, CodeFormatterClang.CLANG_FORMAT_FILE)
|
||
|
|
if os.path.exists(clangFormatFile):
|
||
|
|
if clangFormatFile not in foundClangFormatFiles:
|
||
|
|
foundClangFormatFiles.add(clangFormatFile)
|
||
|
|
if os.path.exists(self.checkedInClangFormatFile) and \
|
||
|
|
not filecmp.cmp(self.checkedInClangFormatFile, clangFormatFile):
|
||
|
|
cprint("[WARN] " + clangFormatFile + " does not match " + self.checkedInClangFormatFile, "yellow")
|
||
|
|
self.confirmWithUserClangFormatFileCantBeVerified()
|
||
|
|
else:
|
||
|
|
print("[OK] Found " + CodeFormatterClang.CLANG_FORMAT_FILE +
|
||
|
|
" file (used by the formatter) " + clangFormatFile)
|
||
|
|
return True
|
||
|
|
else:
|
||
|
|
dirNameOneLevelUp = os.path.dirname(dirName)
|
||
|
|
if dirNameOneLevelUp == dirName:
|
||
|
|
# dirName was already root folder -> clang-format file not found
|
||
|
|
cprint("[ERROR] Not found " + CodeFormatterClang.CLANG_FORMAT_FILE + " for " +
|
||
|
|
fileName + " in same directory or in any parent directory", "red")
|
||
|
|
return False
|
||
|
|
else:
|
||
|
|
return self.findClangFormatFileStartingFrom(dirNameOneLevelUp, fileName, foundClangFormatFiles)
|
||
|
|
|
||
|
|
|
||
|
|
class CodeFormatterAutopep(CodeFormatter):
|
||
|
|
|
||
|
|
def __init__(self):
|
||
|
|
CodeFormatter.__init__(self,
|
||
|
|
command="autopep8",
|
||
|
|
expectedVersion="",
|
||
|
|
formatCommandArguments=["--in-place", "--max-line-length=119"],
|
||
|
|
verifyCommandArguments=["--diff", "--max-line-length=119"],
|
||
|
|
verifyOutputIsDiff=True,
|
||
|
|
fileEndings=["py"],
|
||
|
|
fileDescription="python",
|
||
|
|
installCommand="sudo apt-get install python-pep8 python-autopep8")
|
||
|
|
|
||
|
|
|
||
|
|
class CodeFormat:
|
||
|
|
|
||
|
|
def __init__(self):
|
||
|
|
self.failure = False
|
||
|
|
self.codeFormatterInstances = []
|
||
|
|
return
|
||
|
|
|
||
|
|
def parseCommandLine(self):
|
||
|
|
parser = argparse.ArgumentParser(
|
||
|
|
description="Helper script for code formatting.")
|
||
|
|
parser.add_argument("input", nargs="+",
|
||
|
|
help="files or directories to process")
|
||
|
|
parser.add_argument("-v", "--verify", action="store_true",
|
||
|
|
help="do not change file, but only verify the format is correct")
|
||
|
|
parser.add_argument("-d", "--diff", action="store_true",
|
||
|
|
help="show diff, implies verify mode")
|
||
|
|
parser.add_argument("-e", "--exclude", nargs="+", metavar="exclude",
|
||
|
|
help="exclude files or directories containing words from the exclude list in their names")
|
||
|
|
parser.add_argument("-y", "--yes", action="store_true",
|
||
|
|
help="do not ask for confirmation before formatting more than one file")
|
||
|
|
parser.add_argument("--version", action="version", version="%(prog)s " + SCRIPT_VERSION)
|
||
|
|
self.args = parser.parse_args()
|
||
|
|
if self.args.diff:
|
||
|
|
self.args.verify = True
|
||
|
|
|
||
|
|
def addCodeFormatter(self, codeFormatterInstance):
|
||
|
|
self.codeFormatterInstances.append(codeFormatterInstance)
|
||
|
|
|
||
|
|
def scanForInputFiles(self):
|
||
|
|
for formatterInstance in self.codeFormatterInstances:
|
||
|
|
filePattern = re.compile("^[^.].*\.(" + "|".join(formatterInstance.fileEndings) + ")$")
|
||
|
|
formatterInstance.inputFiles = []
|
||
|
|
for fileOrDirectory in self.args.input:
|
||
|
|
if os.path.exists(fileOrDirectory):
|
||
|
|
formatterInstance.inputFiles.extend(self.scanFileOrDirectory(fileOrDirectory, filePattern))
|
||
|
|
else:
|
||
|
|
cprint("[WARN] Cannot find '" + fileOrDirectory + "'", "yellow")
|
||
|
|
|
||
|
|
def scanFileOrDirectory(self, fileOrDirectory, filePattern):
|
||
|
|
fileList = []
|
||
|
|
if os.path.isdir(fileOrDirectory):
|
||
|
|
for root, directories, fileNames in os.walk(fileOrDirectory):
|
||
|
|
directories[:] = self.filterDirectories(root, directories)
|
||
|
|
for filename in filter(lambda name: filePattern.match(name), fileNames):
|
||
|
|
fullFilename = os.path.join(root, filename)
|
||
|
|
if self.isFileNotExcluded(fullFilename):
|
||
|
|
fileList.append(fullFilename)
|
||
|
|
else:
|
||
|
|
if self.isFileNotExcluded(fileOrDirectory) and (filePattern.match(os.path.basename(fileOrDirectory)) is not None):
|
||
|
|
fileList.append(fileOrDirectory)
|
||
|
|
return fileList
|
||
|
|
|
||
|
|
def filterDirectories(self, root, directories):
|
||
|
|
# Exclude hidden directories and all directories that have a CODE_FORMAT_IGNORE_FILE
|
||
|
|
directories[:] = [directory for directory in directories if
|
||
|
|
not directory.startswith(".") and
|
||
|
|
not os.path.exists(os.path.join(root, directory, CodeFormatterClang.CODE_FORMAT_IGNORE_FILE))]
|
||
|
|
return directories
|
||
|
|
|
||
|
|
def isFileNotExcluded(self, fileName):
|
||
|
|
if self.args.exclude is not None:
|
||
|
|
for excluded in self.args.exclude:
|
||
|
|
if excluded in fileName:
|
||
|
|
return False
|
||
|
|
if os.path.islink(fileName):
|
||
|
|
return False
|
||
|
|
|
||
|
|
return True
|
||
|
|
|
||
|
|
def confirmWithUserFileIsOutsideGit(self, fileName):
|
||
|
|
cprint("[WARN] File is not in a Git repo: " + fileName, "yellow")
|
||
|
|
answer = raw_input("Are you sure to code format it anyway? (y/Q)")
|
||
|
|
if answer != "y":
|
||
|
|
sys.exit(1)
|
||
|
|
|
||
|
|
def confirmWithUserFileIsUntracked(self, fileName):
|
||
|
|
cprint("[WARN] File is untracked in Git: " + fileName, "yellow")
|
||
|
|
answer = raw_input("Are you sure to code format it anyway? (y/Q)")
|
||
|
|
if answer != "y":
|
||
|
|
sys.exit(1)
|
||
|
|
|
||
|
|
def confirmWithUserGitRepoIsNotClean(self, gitRepo):
|
||
|
|
cprint("[WARN] Git repo is not clean: " + gitRepo, "yellow")
|
||
|
|
answer = raw_input("Are you sure to code format files in it anyway? (y/Q)")
|
||
|
|
if answer != "y":
|
||
|
|
sys.exit(1)
|
||
|
|
|
||
|
|
def checkInputFilesAreInCleanGitReposAndAreTracked(self):
|
||
|
|
if self.args.verify or self.args.yes:
|
||
|
|
return
|
||
|
|
gitRepos = sets.Set()
|
||
|
|
for formatterInstance in self.codeFormatterInstances:
|
||
|
|
for fileName in formatterInstance.inputFiles:
|
||
|
|
gitRepo = self.getGitRepoForFile(fileName)
|
||
|
|
if gitRepo is None:
|
||
|
|
self.confirmWithUserFileIsOutsideGit(fileName)
|
||
|
|
else:
|
||
|
|
self.gitUpdateIndexRefresh(gitRepo)
|
||
|
|
if not self.isTrackedFile(fileName):
|
||
|
|
self.confirmWithUserFileIsUntracked(fileName)
|
||
|
|
elif gitRepo not in gitRepos:
|
||
|
|
gitRepos.add(gitRepo)
|
||
|
|
if not self.isCleanGitRepo(gitRepo):
|
||
|
|
self.confirmWithUserGitRepoIsNotClean(gitRepo)
|
||
|
|
|
||
|
|
def getGitRepoForFile(self, fileName):
|
||
|
|
if not self.isInsideGitRepo(fileName):
|
||
|
|
return None
|
||
|
|
try:
|
||
|
|
gitProcess = subprocess.Popen(["git", "rev-parse", "--show-toplevel"],
|
||
|
|
stdin=subprocess.PIPE,
|
||
|
|
stdout=subprocess.PIPE,
|
||
|
|
stderr=subprocess.PIPE,
|
||
|
|
cwd=os.path.dirname(fileName))
|
||
|
|
gitOutput, _ = gitProcess.communicate()
|
||
|
|
if gitProcess.returncode == 0:
|
||
|
|
return gitOutput.rstrip('\r\n')
|
||
|
|
except OSError:
|
||
|
|
cprint("[ERROR] Failed to run 'git rev-parse --show-toplevel' for " + fileName, "red")
|
||
|
|
return None
|
||
|
|
|
||
|
|
def isInsideGitRepo(self, fileName):
|
||
|
|
try:
|
||
|
|
gitProcess = subprocess.Popen(["git", "rev-parse", "--is-inside-work-tree"],
|
||
|
|
stdin=subprocess.PIPE,
|
||
|
|
stdout=subprocess.PIPE,
|
||
|
|
stderr=subprocess.PIPE,
|
||
|
|
cwd=os.path.dirname(fileName))
|
||
|
|
gitOutput, _ = gitProcess.communicate()
|
||
|
|
if gitProcess.returncode == 0:
|
||
|
|
return gitOutput.rstrip('\r\n') == "true"
|
||
|
|
except OSError:
|
||
|
|
cprint("[ERROR] Failed to run 'git rev-parse --is-inside-work-tree' for " + fileName, "red")
|
||
|
|
return False
|
||
|
|
|
||
|
|
def isTrackedFile(self, fileName):
|
||
|
|
try:
|
||
|
|
gitProcess = subprocess.Popen(["git", "ls-files", "--error-unmatch", "--", os.path.basename(fileName)],
|
||
|
|
stdin=subprocess.PIPE,
|
||
|
|
stdout=subprocess.PIPE,
|
||
|
|
stderr=subprocess.PIPE,
|
||
|
|
cwd=os.path.dirname(fileName))
|
||
|
|
_, _ = gitProcess.communicate()
|
||
|
|
if gitProcess.returncode == 0:
|
||
|
|
return True
|
||
|
|
except OSError:
|
||
|
|
cprint("[ERROR] Failed to run 'git ls-files --error-unmatch' for " + fileName, "red")
|
||
|
|
return False
|
||
|
|
|
||
|
|
def isCleanGitRepo(self, gitRepo):
|
||
|
|
try:
|
||
|
|
gitProcess = subprocess.Popen(["git", "diff-index", "--quiet", "HEAD", "--"],
|
||
|
|
stdin=subprocess.PIPE,
|
||
|
|
stdout=subprocess.PIPE,
|
||
|
|
stderr=subprocess.PIPE,
|
||
|
|
cwd=gitRepo)
|
||
|
|
_, _ = gitProcess.communicate()
|
||
|
|
if gitProcess.returncode == 0:
|
||
|
|
return True
|
||
|
|
except OSError:
|
||
|
|
cprint("[ERROR] Failed to run 'git diff-index --quiet HEAD --' for " + gitRepo, "red")
|
||
|
|
return False
|
||
|
|
|
||
|
|
def gitUpdateIndexRefresh(self, gitRepo):
|
||
|
|
try:
|
||
|
|
gitProcess = subprocess.Popen(["git", "update-index", "-q", "--ignore-submodules", "--refresh"],
|
||
|
|
stdin=subprocess.PIPE,
|
||
|
|
stdout=subprocess.PIPE,
|
||
|
|
stderr=subprocess.PIPE,
|
||
|
|
cwd=gitRepo)
|
||
|
|
_, _ = gitProcess.communicate()
|
||
|
|
if gitProcess.returncode == 0:
|
||
|
|
return True
|
||
|
|
except OSError:
|
||
|
|
cprint("[ERROR] Failed to run 'git update-index -q --ignore-submodules --refresh' for " + gitRepo, "red")
|
||
|
|
return False
|
||
|
|
|
||
|
|
def verifyFormatterVersion(self):
|
||
|
|
for formatterInstance in self.codeFormatterInstances:
|
||
|
|
if len(formatterInstance.inputFiles) > 0:
|
||
|
|
formatterInstance.verifyFormatterVersion()
|
||
|
|
|
||
|
|
def printMode(self):
|
||
|
|
if self.args.verify:
|
||
|
|
cprint("VERIFY MODE", attrs=["bold"])
|
||
|
|
else:
|
||
|
|
cprint("FORMAT MODE", attrs=["bold"])
|
||
|
|
|
||
|
|
def processFiles(self):
|
||
|
|
for formatterInstance in self.codeFormatterInstances:
|
||
|
|
for fileName in formatterInstance.inputFiles:
|
||
|
|
if self.args.verify:
|
||
|
|
self.failure |= formatterInstance.verifyFile(fileName, self.args.diff)
|
||
|
|
else:
|
||
|
|
self.failure |= formatterInstance.formatFile(fileName)
|
||
|
|
|
||
|
|
def numberOfInputFiles(self):
|
||
|
|
count = 0
|
||
|
|
for formatterInstance in self.codeFormatterInstances:
|
||
|
|
count += len(formatterInstance.inputFiles)
|
||
|
|
return count
|
||
|
|
|
||
|
|
def confirmWithUser(self):
|
||
|
|
if self.numberOfInputFiles() == 0:
|
||
|
|
cprint("[WARN] No input files (or file endings unknown)", "yellow")
|
||
|
|
elif (not self.args.verify) and (not self.args.yes) and self.numberOfInputFiles() > 1:
|
||
|
|
for formatterInstance in self.codeFormatterInstances:
|
||
|
|
formatterInstance.printInputFiles()
|
||
|
|
answer = raw_input("Are you sure to code format " + str(self.numberOfInputFiles()) + " files? (y/N)")
|
||
|
|
if answer != "y":
|
||
|
|
sys.exit(1)
|
||
|
|
|
||
|
|
|
||
|
|
def main():
|
||
|
|
codeFormat = CodeFormat()
|
||
|
|
codeFormat.parseCommandLine()
|
||
|
|
codeFormat.printMode()
|
||
|
|
|
||
|
|
codeFormat.addCodeFormatter(CodeFormatterClang())
|
||
|
|
codeFormat.addCodeFormatter(CodeFormatterAutopep())
|
||
|
|
|
||
|
|
codeFormat.scanForInputFiles()
|
||
|
|
codeFormat.verifyFormatterVersion()
|
||
|
|
codeFormat.confirmWithUser()
|
||
|
|
codeFormat.checkInputFilesAreInCleanGitReposAndAreTracked()
|
||
|
|
codeFormat.processFiles()
|
||
|
|
if codeFormat.failure:
|
||
|
|
cprint("FAILURE", "red")
|
||
|
|
sys.exit(1)
|
||
|
|
else:
|
||
|
|
cprint("SUCCESS", "green")
|
||
|
|
sys.exit(0)
|
||
|
|
|
||
|
|
if __name__ == "__main__":
|
||
|
|
main()
|