01 /**
02 * BSD-style license; for more info see http://pmd.sourceforge.net/license.html
03 */
04 package net.sourceforge.pmd.cpd;
05
06 import java.io.StringReader;
07
08 import net.sourceforge.pmd.lang.LanguageVersion;
09 import net.sourceforge.pmd.lang.TokenManager;
10 import net.sourceforge.pmd.lang.ast.TokenMgrError;
11 import net.sourceforge.pmd.lang.cpp.ast.Token;
12
13 public class CPPTokenizer implements Tokenizer {
14
15 public void tokenize(SourceCode sourceCode, Tokens tokenEntries) {
16 StringBuffer buffer = sourceCode.getCodeBuffer();
17 try {
18 TokenManager tokenManager = LanguageVersion.CPP.getLanguageVersionHandler().getParser().getTokenManager(
19 sourceCode.getFileName(), new StringReader(buffer.toString()));
20 Token currentToken = (Token) tokenManager.getNextToken();
21 while (currentToken.image.length() > 0) {
22 tokenEntries.add(new TokenEntry(currentToken.image, sourceCode.getFileName(), currentToken.beginLine));
23 currentToken = (Token) tokenManager.getNextToken();
24 }
25 tokenEntries.add(TokenEntry.getEOF());
26 System.err.println("Added " + sourceCode.getFileName());
27 } catch (TokenMgrError err) {
28 err.printStackTrace();
29 System.err.println("Skipping " + sourceCode.getFileName() + " due to parse error");
30 tokenEntries.add(TokenEntry.getEOF());
31 }
32 }
33 }
|