/*----------------------------------------------------------------------------*/ /* */ /* Copyright (c) 1995, 2004 IBM Corporation. All rights reserved. */ /* Copyright (c) 2005-2020 Rexx Language Association. All rights reserved. */ /* */ /* This program and the accompanying materials are made available under */ /* the terms of the Common Public License v1.0 which accompanies this */ /* distribution. A copy is also available at the following address: */ /* http://www.oorexx.org/license.html */ /* */ /* Redistribution and use in source and binary forms, with or */ /* without modification, are permitted provided that the following */ /* conditions are met: */ /* */ /* Redistributions of source code must retain the above copyright */ /* notice, this list of conditions and the following disclaimer. */ /* Redistributions in binary form must reproduce the above copyright */ /* notice, this list of conditions and the following disclaimer in */ /* the documentation and/or other materials provided with the distribution. */ /* */ /* Neither the name of Rexx Language Association nor the names */ /* of its contributors may be used to endorse or promote products */ /* derived from this software without specific prior written permission. */ /* */ /* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS */ /* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT */ /* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS */ /* FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT */ /* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, */ /* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED */ /* TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, */ /* OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY */ /* OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING */ /* NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS */ /* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ /* */ /*----------------------------------------------------------------------------*/ /******************************************************************************/ /* REXX Kernel */ /* */ /* Primitive Translator Token Class Definitions */ /* */ /******************************************************************************/ #ifndef Included_RexxToken #define Included_RexxToken #include "SourceLocation.hpp" class LanguageParser; /** * A class for an entry in the various token keyword tables. */ class KeywordEntry { public: /** * Constructor for a keyword table entry. * * @param n The string name of the keyword. * @param code The associated keyword tag mapping. */ inline KeywordEntry(const char *n, int code) { name = n; length = strlen(name); keywordCode = code; } const char *name; // the keyword name size_t length; // length of the keyword string int keywordCode; // the string-to-identifier mapping }; // major token types typedef enum { TOKEN_NULL = 1, TOKEN_BLANK, TOKEN_SYMBOL, TOKEN_LITERAL, TOKEN_OPERATOR, TOKEN_EOC, TOKEN_COMMA, TOKEN_PREFIX, TOKEN_LEFT, TOKEN_RIGHT, TOKEN_POINT, TOKEN_COLON, TOKEN_TILDE, TOKEN_DTILDE, TOKEN_SQLEFT, TOKEN_SQRIGHT, TOKEN_DCOLON, TOKEN_CONTINUE, TOKEN_ASSIGNMENT, } TokenClass; // token extended types. Each category gets is own // section of the numeric range. typedef enum { SUBTYPE_NONE = 0, // Operator sub-types. // NOTE: The operator subtypes are used for table // lookups, so these need start with the index of // 1. OPERATOR_PLUS = 1, OPERATOR_SUBTRACT, OPERATOR_MULTIPLY, OPERATOR_DIVIDE, OPERATOR_INTDIV, OPERATOR_REMAINDER, OPERATOR_POWER, OPERATOR_ABUTTAL, OPERATOR_CONCATENATE, OPERATOR_BLANK, OPERATOR_EQUAL, OPERATOR_BACKSLASH_EQUAL, OPERATOR_GREATERTHAN, OPERATOR_BACKSLASH_GREATERTHAN, OPERATOR_LESSTHAN, OPERATOR_BACKSLASH_LESSTHAN, OPERATOR_GREATERTHAN_EQUAL, OPERATOR_LESSTHAN_EQUAL, OPERATOR_STRICT_EQUAL, OPERATOR_STRICT_BACKSLASH_EQUAL, OPERATOR_STRICT_GREATERTHAN, OPERATOR_STRICT_BACKSLASH_GREATERTHAN, OPERATOR_STRICT_LESSTHAN, OPERATOR_STRICT_BACKSLASH_LESSTHAN, OPERATOR_STRICT_GREATERTHAN_EQUAL, OPERATOR_STRICT_LESSTHAN_EQUAL, OPERATOR_LESSTHAN_GREATERTHAN, OPERATOR_GREATERTHAN_LESSTHAN, OPERATOR_AND, OPERATOR_OR, OPERATOR_XOR, OPERATOR_BACKSLASH, // symbol characters SYMBOL_CONSTANT = 100, SYMBOL_VARIABLE, SYMBOL_NAME, SYMBOL_COMPOUND, SYMBOL_STEM, SYMBOL_DUMMY, SYMBOL_DOTSYMBOL, INTEGER_CONSTANT, LITERAL_STRING, LITERAL_HEX, LITERAL_BIN, // special clause end types CLAUSEEND_EOF = 200, CLAUSEEND_SEMICOLON, CLAUSEEND_EOL, CLAUSEEND_NULL, } TokenSubclass; // various keyword enumerations // the keyword instruction and directive identifiers // IMPORTANT NOTE: These constants are set into instruction // objects. To preserve release-to-release compiled image // compatibility, new constants need to be added to the end // of this list. typedef enum { KEYWORD_NONE = 0, KEYWORD_ADDRESS, KEYWORD_ARG, KEYWORD_CALL, KEYWORD_DO, KEYWORD_DROP, KEYWORD_EXIT, KEYWORD_IF, KEYWORD_INTERPRET, KEYWORD_ITERATE, KEYWORD_LEAVE, KEYWORD_NOP, KEYWORD_NUMERIC, KEYWORD_OPTIONS, KEYWORD_PARSE, KEYWORD_PROCEDURE, KEYWORD_PULL, KEYWORD_PUSH, KEYWORD_QUEUE, KEYWORD_REPLY, KEYWORD_RETURN, KEYWORD_SAY, KEYWORD_SELECT, KEYWORD_SIGNAL, KEYWORD_TRACE, KEYWORD_GUARD, KEYWORD_USE, KEYWORD_EXPOSE, KEYWORD_RAISE, KEYWORD_ELSE, KEYWORD_THEN, KEYWORD_END, KEYWORD_OTHERWISE, KEYWORD_IFTHEN, KEYWORD_WHENTHEN, KEYWORD_WHEN, KEYWORD_ASSIGNMENT, KEYWORD_COMMAND, KEYWORD_MESSAGE, KEYWORD_LABEL, KEYWORD_ENDIF, KEYWORD_BLOCK, KEYWORD_FIRST, KEYWORD_LAST, KEYWORD_ENDELSE, KEYWORD_ENDTHEN, KEYWORD_ENDWHEN, KEYWORD_INSTRUCTION, KEYWORD_FORWARD, KEYWORD_LOOP, KEYWORD_SIGNAL_VALUE, KEYWORD_SIGNAL_ON, KEYWORD_CALL_VALUE, KEYWORD_CALL_QUALIFIED, KEYWORD_CALL_ON, KEYWORD_SIMPLE_BLOCK, KEYWORD_LOOP_FOREVER, KEYWORD_LOOP_OVER, KEYWORD_LOOP_OVER_UNTIL, KEYWORD_LOOP_OVER_WHILE, KEYWORD_LOOP_OVER_FOR, KEYWORD_LOOP_OVER_FOR_UNTIL, KEYWORD_LOOP_OVER_FOR_WHILE, KEYWORD_LOOP_CONTROLLED, KEYWORD_LOOP_CONTROLLED_UNTIL, KEYWORD_LOOP_CONTROLLED_WHILE, KEYWORD_LOOP_COUNT, KEYWORD_LOOP_COUNT_UNTIL, KEYWORD_LOOP_COUNT_WHILE, KEYWORD_LOOP_WHILE, KEYWORD_LOOP_UNTIL, KEYWORD_LOOP_WITH, KEYWORD_LOOP_WITH_UNTIL, KEYWORD_LOOP_WITH_WHILE, KEYWORD_LOOP_WITH_FOR, KEYWORD_LOOP_WITH_FOR_UNTIL, KEYWORD_LOOP_WITH_FOR_WHILE, KEYWORD_MESSAGE_DOUBLE, KEYWORD_SELECT_CASE, KEYWORD_WHEN_CASE, KEYWORD_CLASS, KEYWORD_LIBRARY, KEYWORD_REQUIRES, KEYWORD_USE_LOCAL, KEYWORD_CONSTANT, } InstructionKeyword; // instruction subkeyword types typedef enum { SUBKEY_NONE = 0, SUBKEY_ARG, SUBKEY_BY, SUBKEY_DIGITS, SUBKEY_COUNTER, SUBKEY_END, SUBKEY_ELSE, SUBKEY_ENGINEERING, SUBKEY_EXPOSE, SUBKEY_FOR, SUBKEY_FOREVER, SUBKEY_FORM, SUBKEY_FUZZ, SUBKEY_INDEX, SUBKEY_ITEM, SUBKEY_LINEIN, SUBKEY_LOCAL, SUBKEY_LOWER, SUBKEY_CASELESS, SUBKEY_NAME, SUBKEY_NOVALUE, SUBKEY_OFF, SUBKEY_ON, SUBKEY_OTHERWISE, SUBKEY_OVER, SUBKEY_PULL, SUBKEY_SCIENTIFIC, SUBKEY_SOURCE, SUBKEY_THEN, SUBKEY_TO, SUBKEY_UNTIL, SUBKEY_UPPER, SUBKEY_VALUE, SUBKEY_VAR, SUBKEY_VERSION, SUBKEY_WHEN, SUBKEY_WHILE, SUBKEY_WITH, SUBKEY_DESCRIPTION, SUBKEY_ADDITIONAL, SUBKEY_RESULT, SUBKEY_ARRAY, SUBKEY_RETURN, SUBKEY_EXIT, SUBKEY_CONTINUE, SUBKEY_CLASS, SUBKEY_MESSAGE, SUBKEY_ARGUMENTS, SUBKEY_LABEL, SUBKEY_STRICT, SUBKEY_TRUE, SUBKEY_FALSE, SUBKEY_CASE, SUBKEY_INPUT, SUBKEY_OUTPUT, SUBKEY_ERROR, SUBKEY_STEM, SUBKEY_STREAM, SUBKEY_USING, SUBKEY_NORMAL, SUBKEY_APPEND, SUBKEY_REPLACE, } InstructionSubKeyword; /** * Keyword identifiers for directives. */ typedef enum { DIRECTIVE_NONE = 0, DIRECTIVE_METHOD, DIRECTIVE_OPTIONS, DIRECTIVE_REQUIRES, DIRECTIVE_ROUTINE, DIRECTIVE_CLASS, DIRECTIVE_ATTRIBUTE, DIRECTIVE_LIBRARY, DIRECTIVE_CONSTANT, DIRECTIVE_RESOURCE, DIRECTIVE_ANNOTATE, } DirectiveKeyword; // Identify different token types returned by // locateToken() typedef enum { NORMAL_CHAR, SIGNIFICANT_BLANK, CLAUSE_EOF, CLAUSE_EOL, } CharacterClass; /** * Keywords allowed on directive statements. */ typedef enum { SUBDIRECTIVE_NONE = 0, SUBDIRECTIVE_PUBLIC, SUBDIRECTIVE_METACLASS, SUBDIRECTIVE_INHERIT, SUBDIRECTIVE_PRIVATE, SUBDIRECTIVE_GUARDED, SUBDIRECTIVE_CLASS, SUBDIRECTIVE_EXTERNAL, SUBDIRECTIVE_SUBCLASS, SUBDIRECTIVE_UNGUARDED, SUBDIRECTIVE_MIXINCLASS, SUBDIRECTIVE_ATTRIBUTE, SUBDIRECTIVE_PROTECTED, SUBDIRECTIVE_ABSTRACT, SUBDIRECTIVE_UNPROTECTED, SUBDIRECTIVE_GET, SUBDIRECTIVE_SET, SUBDIRECTIVE_LIBRARY, SUBDIRECTIVE_DIGITS, SUBDIRECTIVE_FORM, SUBDIRECTIVE_FUZZ, SUBDIRECTIVE_TRACE, SUBDIRECTIVE_NAMESPACE, SUBDIRECTIVE_END, SUBDIRECTIVE_NOVALUE, SUBDIRECTIVE_ERROR, SUBDIRECTIVE_CONDITION, SUBDIRECTIVE_PROLOG, SUBDIRECTIVE_NOPROLOG, SUBDIRECTIVE_PACKAGE, SUBDIRECTIVE_METHOD, SUBDIRECTIVE_ROUTINE, SUBDIRECTIVE_CONSTANT, SUBDIRECTIVE_DELEGATE, SUBDIRECTIVE_SYNTAX, SUBDIRECTIVE_ALL, SUBDIRECTIVE_FAILURE, SUBDIRECTIVE_LOSTDIGITS, SUBDIRECTIVE_NOSTRING, SUBDIRECTIVE_NOTREADY, } DirectiveSubKeyword; /** * Condition keyword posibilities */ typedef enum { CONDITION_NONE = 0, CONDITION_ANY, CONDITION_ERROR, CONDITION_FAILURE, CONDITION_HALT, CONDITION_NOMETHOD, CONDITION_NOSTRING, CONDITION_NOTREADY, CONDITION_NOVALUE, CONDITION_PROPAGATE, CONDITION_SYNTAX, CONDITION_USER, CONDITION_LOSTDIGITS } ConditionKeyword; // markers for the builtin function typedef enum { NO_BUILTIN = 0, BUILTIN_ABBREV, BUILTIN_ABS, BUILTIN_ADDRESS, BUILTIN_ARG, BUILTIN_B2X, BUILTIN_BITAND, BUILTIN_BITOR, BUILTIN_BITXOR, BUILTIN_C2D, BUILTIN_C2X, BUILTIN_CENTER, BUILTIN_CENTRE, BUILTIN_CHANGESTR, BUILTIN_CHARIN, BUILTIN_CHAROUT, BUILTIN_CHARS, BUILTIN_COMPARE, BUILTIN_CONDITION, BUILTIN_COPIES, BUILTIN_COUNTSTR, BUILTIN_D2C, BUILTIN_D2X, BUILTIN_DATATYPE, BUILTIN_DATE, BUILTIN_DELSTR, BUILTIN_DELWORD, BUILTIN_DIGITS, BUILTIN_ERRORTEXT, BUILTIN_FORM, BUILTIN_FORMAT, BUILTIN_FUZZ, BUILTIN_INSERT, BUILTIN_LASTPOS, BUILTIN_LEFT, BUILTIN_LENGTH, BUILTIN_LINEIN, BUILTIN_LINEOUT, BUILTIN_LINES, BUILTIN_MAX, BUILTIN_MIN, BUILTIN_OVERLAY, BUILTIN_POS, BUILTIN_QUEUED, BUILTIN_RANDOM, BUILTIN_REVERSE, BUILTIN_RIGHT, BUILTIN_RXQUEUE, BUILTIN_SIGN, BUILTIN_SOURCELINE, BUILTIN_SPACE, BUILTIN_STREAM, BUILTIN_STRIP, BUILTIN_SUBSTR, BUILTIN_SUBWORD, BUILTIN_SYMBOL, BUILTIN_TIME, BUILTIN_TRACE, BUILTIN_TRANSLATE, BUILTIN_TRUNC, BUILTIN_VALUE, BUILTIN_VAR, BUILTIN_VERIFY, BUILTIN_WORD, BUILTIN_WORDINDEX, BUILTIN_WORDLENGTH, BUILTIN_WORDPOS, BUILTIN_WORDS, BUILTIN_X2B, BUILTIN_X2C, BUILTIN_X2D, BUILTIN_XRANGE, BUILTIN_USERID, BUILTIN_LOWER, BUILTIN_UPPER, BUILTIN_RXFUNCADD, BUILTIN_RXFUNCDROP, BUILTIN_RXFUNCQUERY, BUILTIN_ENDLOCAL, BUILTIN_SETLOCAL, BUILTIN_QUALIFY, } BuiltinCode; /******************************************************************************/ /* various expression terminator sets */ /******************************************************************************/ #define TERM_EOC 0x00000001u // terminate on end of clause #define TERM_RIGHT 0x00000002u // terminate on left paren #define TERM_SQRIGHT 0x00000004u // terminate on left square bracket #define TERM_TO 0x00000008u // terminate on TO keyword #define TERM_BY 0x00000010u // terminate on BY keyword #define TERM_FOR 0x00000020u // terminate on FOR keyword #define TERM_WHILE 0x00000040u // terminate on WHILE/UNTIL keywords #define TERM_WITH 0x00000100u // terminate on WITH keyword #define TERM_THEN 0x00000200u // terminate on THEN keyword #define TERM_KEYWORD 0x10000000u // perform keyword terminator checks // terminate on controlled DO keywords #define TERM_CONTROL (TERM_KEYWORD | TERM_TO | TERM_BY | TERM_FOR | TERM_WHILE) // terminate on DO conditionals #define TERM_COND (TERM_KEYWORD | TERM_WHILE) // termantors for an OVER condition #define TERM_OVER (TERM_KEYWORD | TERM_FOR | TERM_WHILE) // terminate on a THEN keyword #define TERM_IF (TERM_KEYWORD | TERM_THEN) class RexxToken : public RexxInternalObject { public: void *operator new(size_t); inline void operator delete(void *) { ; } inline RexxToken() { } inline RexxToken(TokenClass c, SourceLocation &l, TokenSubclass sc = SUBTYPE_NONE, RexxString *v = OREF_NULL) : classId(c), subclass(sc), stringValue(v), numeric(SUBTYPE_NONE), tokenLocation(l) { }; inline RexxToken(RESTORETYPE restoreType) { ; }; void live(size_t) override; void liveGeneral(MarkReason reason) override; inline void setStart(size_t l, size_t o) { tokenLocation.setStart(l, o); } inline void setEnd(size_t l, size_t o) { tokenLocation.setEnd(l, o); } inline size_t getLineNumber() { return tokenLocation.getLineNumber(); } inline bool isType(TokenClass t) { return classId == t; } inline bool isType(TokenClass t1, TokenClass t2) { return classId == t1 || classId == t2; } inline bool isType(TokenClass t1, TokenClass t2, TokenClass t3) { return classId == t1 || classId == t2 || classId == t3; } inline bool isSubtype(TokenSubclass t) { return subclass == t; } inline bool isSubtype(TokenSubclass t1, TokenSubclass t2) { return subclass == t1 || subclass == t2; } inline TokenClass type() { return classId; } inline TokenSubclass subtype() { return subclass; } inline RexxString *value() { return stringValue; } RexxString *upperValue(); inline void setType(TokenClass t) { classId = t; } inline void setSubtype(TokenSubclass t) { subclass = t; } inline void setValue(RexxString *v) { stringValue = v; } inline bool isVariable() { return (subclass == SYMBOL_VARIABLE || subclass == SYMBOL_STEM || subclass == SYMBOL_COMPOUND); }; inline bool isStem() { return (subclass == SYMBOL_STEM); } inline bool isNonCompoundVariable() { return subclass == SYMBOL_VARIABLE || subclass == SYMBOL_STEM; }; inline bool isSimpleVariable() { return subclass == SYMBOL_VARIABLE; }; inline bool isVariableOrDot() { return isVariable() || subclass == SYMBOL_DOTSYMBOL; }; inline bool isDotSymbol() { return (subclass == SYMBOL_DOTSYMBOL); } inline bool isDot() { return (subclass == SYMBOL_DUMMY); } inline bool isLiteral() { return classId == TOKEN_LITERAL; }; inline bool isSymbolOrLiteral() { return classId == TOKEN_LITERAL || this->classId == TOKEN_SYMBOL; }; inline bool isConstant() { return (classId == TOKEN_SYMBOL && subclass != SYMBOL_VARIABLE && subclass != SYMBOL_STEM && subclass != SYMBOL_COMPOUND); }; inline bool isNumericSymbol() { return (subclass == SYMBOL_CONSTANT); } inline bool isIntegerConstant() { return numeric == INTEGER_CONSTANT; } inline bool isSymbol() { return classId == TOKEN_SYMBOL; }; inline bool isOperator() { return classId == TOKEN_OPERATOR; } inline bool isOperator(TokenSubclass t) { return isType(TOKEN_OPERATOR) && isSubtype(t); } inline bool isBlank() { return classId == TOKEN_BLANK; } inline bool isComma() { return classId == TOKEN_COMMA; } inline bool isLeftParen() { return classId == TOKEN_LEFT; } inline bool isRightParen() { return classId == TOKEN_RIGHT; } inline bool isLeftBracket() { return classId == TOKEN_SQLEFT; } inline bool isRightBracket() { return classId == TOKEN_SQRIGHT; } inline bool isEndOfClause() { return classId == TOKEN_EOC; } inline bool isEndOfFile() { return isEndOfClause() && isSubtype(CLAUSEEND_EOF); } inline bool isBlankSignificant() { return (classId == TOKEN_SYMBOL || classId == TOKEN_LITERAL || classId == TOKEN_RIGHT || classId == TOKEN_SQRIGHT); } inline bool isMessageOperator() { return classId == TOKEN_TILDE || classId == TOKEN_DTILDE || classId == TOKEN_SQLEFT; } inline void setNumeric(TokenSubclass v) { this->numeric = v; }; inline const SourceLocation &getLocation() { return tokenLocation; } inline void setLocation(const SourceLocation &l) { tokenLocation = l; } void checkAssignment(LanguageParser *parser, RexxString *newValue); int precedence(); bool isTerminator(int); RexxString *displayValue(); InstructionKeyword keyword(); InstructionSubKeyword subKeyword(); BuiltinCode builtin(); ConditionKeyword condition(); InstructionSubKeyword parseOption(); DirectiveKeyword keyDirective(); DirectiveSubKeyword subDirective(); static BuiltinCode resolveBuiltin(RexxString *value); protected: static int resolveKeyword(RexxString *token, KeywordEntry *table, size_t table_size); RexxString *stringValue; // token string value TokenClass classId; // class of token TokenSubclass subclass; // specialized type of token TokenSubclass numeric; // even further specialization SourceLocation tokenLocation; // token source location // Tables of different keywords using in various contexts. static KeywordEntry directives[]; static KeywordEntry keywordInstructions[]; static KeywordEntry subKeywords[]; static KeywordEntry builtinFunctions[]; static KeywordEntry conditionKeywords[]; static KeywordEntry parseOptions[]; static KeywordEntry subDirectives[]; }; inline RexxToken *new_token(TokenClass c, TokenSubclass s, RexxString *v, SourceLocation &l) { return new RexxToken (c, l, s, v); } #endif