import java.io.*;

class Token {
    public String type;     // token type: Identifier, Keyword, Literal, 
                          //             Separator, Operator, or Other
    public String value;    // token value
}

public class TokenStream {

    private boolean isEof = false;
    private char nextChar = ' '; // next character in input stream
    private BufferedReader input;

// Pass a filename for the program text as a source for the TokenStream
    public TokenStream (String fileName) { 
        try {
            input = new BufferedReader (new FileReader(fileName));
        }
        catch (FileNotFoundException e) {
            System.out.println("File not found: " + fileName);
            System.exit(1);
        }
    }

    public Token nextToken() { // Return next token type and value
        Token t = new Token();   
        t.type = "Other";
        t.value = "";

        // first check for whitespace and bypass it
        while (isWhiteSpace(nextChar)) {
            nextChar = readChar();
        }

        // Then check for a comment, and bypass it
        // but remember that / is also a division operator
        if (nextChar=='/') {
            ...
        }
        
        // Then check for an Operator; recover 2-character operators
        // as well as 1-character ones
        if (isOperator(nextChar)) {
            t.type = "Operator";
            ...
        }

        // Then check for a Separator
        if (isSeparator(nextChar)) {
            t.type = "Separator";
            ...
        }
   
        // Then check for an Identifier, Keyword, or Literal
        if (isLetter(nextChar)) { // get an Identifier
            t.type = "Identifier";
            ...
        }
        if (isDigit(nextChar)) { // check for integers
            t.type = "Literal";
            ...
        }
        ...
        return t;
    }

    private char readChar () {
        int i = 0;
        if (isEof) return (char)0;
        System.out.flush();
        try { 
            i = input.read(); 
        } catch (IOException e) {
            System.exit(-1);
        }
        if (i == -1) {
            isEof=true; 
            return (char)0;
        }
        return (char)i;
    }

    private boolean isKeyword (String s) {
        return ...;
    }
  
    private boolean isWhiteSpace (char c) {
        return ...;
    }

    private boolean isEndOfLine (char c) {
        return ...;
    }

    private boolean isSeparator(char c) {
        return (c=='[' || c==']' || c=='(' || c==')' || c==':' ||
	    c=='{' || c=='}' || c==';' || c==',' || c=='.');
    }
  
    private boolean isOperator(char c) {
        return ...;
    }
  
    private boolean isLetter(char c) {
        return ...;
    }
  
    private boolean isDigit(char c) {
        return ...;
    }

    public boolean eof () {
        return isEof;
    }
}