01 /**
02 * BSD-style license; for more info see http://pmd.sourceforge.net/license.html
03 */
04 package net.sourceforge.pmd.cpd;
05
06 import java.io.StringReader;
07 import java.util.Properties;
08
09 import net.sourceforge.pmd.lang.LanguageVersion;
10 import net.sourceforge.pmd.lang.TokenManager;
11 import net.sourceforge.pmd.lang.java.ast.JavaParserConstants;
12 import net.sourceforge.pmd.lang.java.ast.Token;
13
14 public class JavaTokenizer implements Tokenizer {
15
16 public static final String IGNORE_LITERALS = "ignore_literals";
17 public static final String IGNORE_IDENTIFIERS = "ignore_identifiers";
18
19 private boolean ignoreLiterals;
20 private boolean ignoreIdentifiers;
21
22 public void setProperties(Properties properties) {
23 ignoreLiterals = Boolean.parseBoolean(properties.getProperty(IGNORE_LITERALS, "false"));
24 ignoreIdentifiers = Boolean.parseBoolean(properties.getProperty(IGNORE_IDENTIFIERS, "false"));
25 }
26
27 public void tokenize(SourceCode sourceCode, Tokens tokenEntries) {
28 StringBuffer buffer = sourceCode.getCodeBuffer();
29
30 /*
31 I'm doing a sort of State pattern thing here where
32 this goes into "discarding" mode when it hits an import or package
33 keyword and goes back into "accumulate mode" when it hits a semicolon.
34 This could probably be turned into some objects.
35 */
36 // Note that Java version is irrelevant for tokenizing
37 TokenManager tokenMgr = LanguageVersion.JAVA_14.getLanguageVersionHandler().getParser().getTokenManager(
38 sourceCode.getFileName(), new StringReader(buffer.toString()));
39 Token currentToken = (Token) tokenMgr.getNextToken();
40 boolean inDiscardingState = false;
41 while (currentToken.image.length() > 0) {
42 if (currentToken.kind == JavaParserConstants.IMPORT || currentToken.kind == JavaParserConstants.PACKAGE) {
43 inDiscardingState = true;
44 currentToken = (Token) tokenMgr.getNextToken();
45 continue;
46 }
47
48 if (inDiscardingState && currentToken.kind == JavaParserConstants.SEMICOLON) {
49 inDiscardingState = false;
50 }
51
52 if (inDiscardingState) {
53 currentToken = (Token) tokenMgr.getNextToken();
54 continue;
55 }
56
57 if (currentToken.kind != JavaParserConstants.SEMICOLON) {
58 String image = currentToken.image;
59 if (ignoreLiterals
60 && (currentToken.kind == JavaParserConstants.STRING_LITERAL
61 || currentToken.kind == JavaParserConstants.CHARACTER_LITERAL
62 || currentToken.kind == JavaParserConstants.DECIMAL_LITERAL || currentToken.kind == JavaParserConstants.FLOATING_POINT_LITERAL)) {
63 image = String.valueOf(currentToken.kind);
64 }
65 if (ignoreIdentifiers && currentToken.kind == JavaParserConstants.IDENTIFIER) {
66 image = String.valueOf(currentToken.kind);
67 }
68 tokenEntries.add(new TokenEntry(image, sourceCode.getFileName(), currentToken.beginLine));
69 }
70
71 currentToken = (Token) tokenMgr.getNextToken();
72 }
73 tokenEntries.add(TokenEntry.getEOF());
74 }
75
76 public void setIgnoreLiterals(boolean ignore) {
77 this.ignoreLiterals = ignore;
78 }
79
80 public void setIgnoreIdentifiers(boolean ignore) {
81 this.ignoreIdentifiers = ignore;
82 }
83 }
|