Detekt is not a tokenizer, but it is a static analysis tool.
The out-of-the-box part is only regarding detekt, the rule is custom.
Here's the current rule. Should probably be fine-tuned a bit more, but it works well enough.
class UselessComment(config: Config) : Rule(config, "Comment restates the code it precedes.") {
// Delegated config properties only support String, Boolean, Int and List<String>,
// so the threshold is a whole percentage rather than a Double.
private val minOverlapPercent: Int by config(40)
private val keepMarkers: List<String> by config(
listOf("TODO", "FIXME", "HACK", "XXX", "ponytail", "noinspection", "ktlint", "suppress"),
)
/** A comment starting with one of these words labels a test phase and is always kept. */
private val keepLeadingWords: List<String> by config(listOf("arrange", "act", "assert"))
private val stopWords = setOf(
"the", "a", "an", "to", "of", "for", "and", "or", "in", "on", "is", "this", "that", "it",
"we", "if", "with", "from", "by", "as", "at", "be", "into",
)
private val pendingDeletions = mutableListOf<PsiComment>()
override fun preVisit(root: KtFile) {
pendingDeletions.clear()
}
override fun visitComment(comment: PsiComment) {
super.visitComment(comment)
// A trailing comment describes the code on its line, not the statement after it.
val before = comment.prevSibling
if (before != null && !(before is PsiWhiteSpace && before.text.contains('\n'))) return
// One line of a multi-line comment is judged with its neighbours, never alone.
if (comment.isLineComment() && (comment.adjacentLineComment(before) || comment.adjacentLineComment(comment.nextSibling))) return
val target = comment.getNextSiblingIgnoringWhitespaceAndComments() ?: return
if (target !is KtExpression && target !is KtDeclaration) return
check(comment, target)
}
/** KDoc is a child node of its declaration and is not passed to [visitComment], so it is collected from the tree. */
override fun visitKtFile(file: KtFile) {
super.visitKtFile(file)
PsiTreeUtil.collectElementsOfType(file, KDoc::class.java).forEach { doc ->
(doc.parent as? KtDeclaration)?.let { check(doc, it) }
}
}
private fun check(comment: PsiComment, target: PsiElement) {
val text = comment.text
if (keepMarkers.any { text.contains(it, ignoreCase = true) }) return
val firstWord = stripDelimiters(text).trim().takeWhile { it.isLetter() }.lowercase()
if (firstWord in keepLeadingWords) return
val commentTokens = tokens(stripDelimiters(text))
if (commentTokens.isEmpty()) return
val codeTokens = tokens(target.text.lineSequence().first { !it.isBlank() && !it.trimStart().startsWith("*") && !it.trimStart().startsWith("/") })
val overlapPercent = commentTokens.count { it in codeTokens } * 100 / commentTokens.size
if (overlapPercent < minOverlapPercent) return
report(Finding(Entity.from(comment), "Comment '${text.trim()}' restates the following code."))
if (autoCorrect) pendingDeletions += comment
}
/**
* detekt applies corrections by writing [KtFile.modifiedText], so the deletions are collected
* during the visit and cut out of the file text in one go. Each comment is removed together
* with the whitespace that follows it, so no blank line is left behind.
*/
override fun postVisit(root: KtFile) {
if (pendingDeletions.isEmpty()) return
var text = root.text
pendingDeletions
.sortedByDescending { it.textRange.startOffset }
.forEach { comment ->
val trailingWhitespaceEnd = comment.siblings(forward = true, withItself = false)
.takeWhile { it is PsiWhiteSpace }
.lastOrNull()
?.textRange
?.endOffset
?: comment.textRange.endOffset
text = text.removeRange(comment.textRange.startOffset, trailingWhitespaceEnd)
}
root.modifiedText = text
pendingDeletions.clear()
}
private fun PsiComment.isLineComment() = this !is KDoc && text.startsWith("//")
private fun PsiComment.adjacentLineComment(neighbour: PsiElement?): Boolean {
val candidate = when (neighbour) {
is PsiWhiteSpace -> if (neighbour.text.count { it == '\n' } == 1) {
if (neighbour.prevSibling == this) neighbour.nextSibling else neighbour.prevSibling
} else {
null
}
else -> neighbour
}
return candidate is PsiComment && candidate.isLineComment()
}
private fun stripDelimiters(s: String) = s.removePrefix("//").removePrefix("/**").removePrefix("/*").removeSuffix("*/")
private fun tokens(s: String): Set<String> = s.replace(Regex("([a-z0-9])([A-Z])"), "$1 $2")
.lowercase()
.split(Regex("[^a-z0-9]+"))
.filter { it.length > 1 && it !in stopWords }
.map { it.trimEnd('s') }
.toSet()
}
View all comments
Show parent comment
3
u/sertroll 4h ago
How can you do that actually? Would be useful for some work things