1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22 package ch.qos.cal10n.util;
23
24 import java.io.BufferedReader;
25 import java.io.IOException;
26 import java.io.Reader;
27 import java.util.ArrayList;
28 import java.util.List;
29
30 import ch.qos.cal10n.MessageConveyorException;
31 import ch.qos.cal10n.util.Token.TokenType;
32
33
34
35
36
37
38 public class TokenStream {
39
40 enum State {
41 START, COMMENT, KEY, SEPARATOR, VAL, TRAILING_BACKSLASH;
42 }
43
44 BufferedReader lineReader;
45 State state = State.START;
46
47 TokenStream(Reader reader) {
48 this.lineReader = new BufferedReader(reader);
49 }
50
51 List<Token> tokenize() {
52 List<Token> tokenList = new ArrayList<Token>();
53
54 while (true) {
55 String currentLine;
56 try {
57 currentLine = lineReader.readLine();
58 } catch (IOException e) {
59 throw new MessageConveyorException("Failed to read input stream", e);
60 }
61 if (currentLine == null) {
62 break;
63 }
64 if(state != State.TRAILING_BACKSLASH) {
65 state = State.START;
66 }
67 tokenizeLine(tokenList, currentLine);
68 tokenList.add(Token.EOL);
69 }
70
71 return tokenList;
72 }
73
74 private void tokenizeLine(List<Token> tokenList, String line) {
75 int len = line.length();
76 StringBuffer buf = new StringBuffer();
77
78 for (int pointer = 0; pointer < len; pointer++) {
79 char c = line.charAt(pointer);
80 switch (state) {
81 case START:
82 if (isWhiteSpace(c)) {
83
84 } else if (c == '#') {
85 state = State.COMMENT;
86 return;
87 } else if (isNonWhiteSpaceSeparator(c)) {
88 state = State.SEPARATOR;
89 buf.append(c);
90 } else {
91 state = State.KEY;
92 buf.append(c);
93 }
94 break;
95
96 case KEY:
97 if (isWhiteSpace(c) || isNonWhiteSpaceSeparator(c)) {
98 tokenList.add(new Token(TokenType.KEY, buf.toString()));
99 buf.setLength(0);
100 buf.append(c);
101 state = State.SEPARATOR;
102 } else {
103 buf.append(c);
104 }
105 break;
106
107 case SEPARATOR:
108
109 if (isWhiteSpace(c) || isNonWhiteSpaceSeparator(c)) {
110 buf.append(c);
111 } else {
112 tokenList.add(new Token(TokenType.SEPARATOR, buf.toString()));
113 buf.setLength(0);
114 buf.append(c);
115 state = State.VAL;
116 }
117 break;
118
119 case VAL:
120 if(c == '\\') {
121 if(isTrailingBackSlash(line, pointer+1)) {
122 tokenList.add(new Token(TokenType.VALUE, buf.toString()));
123 buf.setLength(0);
124 state = State.TRAILING_BACKSLASH;
125 tokenList.add(Token.TRAILING_BACKSLASH);
126 return;
127 } else {
128 buf.append(c);
129 }
130 } else {
131 buf.append(c);
132 }
133 break;
134
135 case TRAILING_BACKSLASH:
136 if (!isWhiteSpace(c)) {
137 buf.append(c);
138 state = State.VAL;
139 }
140 }
141 }
142
143 if(state == State.VAL) {
144 tokenList.add(new Token(TokenType.VALUE, buf.toString()));
145 buf.setLength(0);
146 }
147 }
148
149 boolean isTrailingBackSlash(String line, int next) {
150 int len = line.length();
151 for(int i = next; i < len; i++) {
152 char c = line.charAt(i);
153 if(!isWhiteSpace(c))
154 return false;
155 }
156 return true;
157 }
158
159 boolean isWhiteSpace(char c) {
160 switch (c) {
161 case ' ':
162 case '\t':
163 return true;
164 default:
165 return false;
166 }
167 }
168
169 boolean isNonWhiteSpaceSeparator(char c) {
170 switch (c) {
171 case ':':
172 case '=':
173 return true;
174 default:
175 return false;
176 }
177 }
178 }