View Javadoc

1   /*
2    * Copyright (c) 2009 QOS.ch All rights reserved.
3    * 
4    * Permission is hereby granted, free of charge, to any person obtaining a copy
5    * of this software and associated documentation files (the "Software"), to deal
6    * in the Software without restriction, including without limitation the rights
7    * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
8    * copies of the Software, and to permit persons to whom the Software is
9    * furnished to do so, subject to the following conditions:
10   * 
11   * The above copyright notice and this permission notice shall be included in
12   * all copies or substantial portions of the Software.
13   * 
14   * THE SOFTWARE IS PROVIDED "AS  IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15   * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16   * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
17   * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18   * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
19   * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
20   * SOFTWARE.
21   */
22  package ch.qos.cal10n.util;
23  
24  import java.io.BufferedReader;
25  import java.io.IOException;
26  import java.io.Reader;
27  import java.util.ArrayList;
28  import java.util.List;
29  
30  import ch.qos.cal10n.MessageConveyorException;
31  import ch.qos.cal10n.util.Token.TokenType;
32  
33  /**
34   * 
35   * @author Ceki Gülcü
36   *
37   */
38  public class TokenStream {
39  
40    enum State {
41      START, COMMENT, KEY, SEPARATOR, VAL, TRAILING_BACKSLASH;
42    }
43  
44    BufferedReader lineReader;
45    State state = State.START;
46  
47    TokenStream(Reader reader) {
48      this.lineReader = new BufferedReader(reader);
49    }
50  
51    List<Token> tokenize() {
52      List<Token> tokenList = new ArrayList<Token>();
53  
54      while (true) {
55        String currentLine;
56        try {
57          currentLine = lineReader.readLine();
58        } catch (IOException e) {
59          throw new MessageConveyorException("Failed to read input stream", e);
60        }
61        if (currentLine == null) {
62          break;
63        }
64        if(state != State.TRAILING_BACKSLASH) {
65          state = State.START;
66        }
67        tokenizeLine(tokenList, currentLine);
68        tokenList.add(Token.EOL);
69      }
70  
71      return tokenList;
72    }
73  
74    private void tokenizeLine(List<Token> tokenList, String line) {
75      int len = line.length();
76      StringBuffer buf = new StringBuffer();
77  
78      for (int pointer = 0; pointer < len; pointer++) {
79        char c = line.charAt(pointer);
80        switch (state) {
81        case START:
82          if (isWhiteSpace(c)) {
83            // same sate
84          } else if (c == '#') {
85            state = State.COMMENT;
86            return;
87          } else if (isNonWhiteSpaceSeparator(c)) {
88            state = State.SEPARATOR;
89            buf.append(c);
90          } else {
91            state = State.KEY;
92            buf.append(c);
93          }
94          break;
95  
96        case KEY:
97          if (isWhiteSpace(c) || isNonWhiteSpaceSeparator(c)) {
98            tokenList.add(new Token(TokenType.KEY, buf.toString()));
99            buf.setLength(0);
100           buf.append(c);
101           state = State.SEPARATOR;
102         } else {
103           buf.append(c);
104         }
105         break;
106         
107       case SEPARATOR:
108 
109         if (isWhiteSpace(c) || isNonWhiteSpaceSeparator(c)) {
110           buf.append(c);
111         } else {
112           tokenList.add(new Token(TokenType.SEPARATOR, buf.toString()));
113           buf.setLength(0);
114           buf.append(c);
115           state = State.VAL;
116         } 
117         break;
118 
119       case VAL:
120         if(c == '\\') {
121           if(isTrailingBackSlash(line, pointer+1)) {
122             tokenList.add(new Token(TokenType.VALUE, buf.toString()));
123             buf.setLength(0);
124             state = State.TRAILING_BACKSLASH;
125             tokenList.add(Token.TRAILING_BACKSLASH);
126             return;
127           } else {
128             buf.append(c);
129           }
130         } else {
131           buf.append(c);
132         }
133         break;
134         
135       case TRAILING_BACKSLASH:
136         if (!isWhiteSpace(c)) {
137           buf.append(c);
138           state = State.VAL;
139         }
140       }
141     }
142     
143     if(state == State.VAL) {
144       tokenList.add(new Token(TokenType.VALUE, buf.toString()));
145       buf.setLength(0);
146     }
147   }
148   
149   boolean isTrailingBackSlash(String line, int next) {
150     int len = line.length();
151     for(int i = next; i < len; i++) {
152       char c = line.charAt(i);
153       if(!isWhiteSpace(c)) 
154         return false;
155     }
156     return true; 
157   }
158 
159   boolean isWhiteSpace(char c) {
160     switch (c) {
161     case ' ':
162     case '\t':
163       return true;
164     default:
165       return false;
166     }
167   }
168 
169   boolean isNonWhiteSpaceSeparator(char c) {
170     switch (c) {
171     case ':':
172     case '=':
173       return true;
174     default:
175       return false;
176     }
177   }
178 }