/*----------------------------------------------------------------------------*/ /* */ /* Copyright (c) 1995, 2004 IBM Corporation. All rights reserved. */ /* Copyright (c) 2005-2025 Rexx Language Association. All rights reserved. */ /* */ /* This program and the accompanying materials are made available under */ /* the terms of the Common Public License v1.0 which accompanies this */ /* distribution. A copy is also available at the following address: */ /* https://www.oorexx.org/license.html */ /* */ /* Redistribution and use in source and binary forms, with or */ /* without modification, are permitted provided that the following */ /* conditions are met: */ /* */ /* Redistributions of source code must retain the above copyright */ /* notice, this list of conditions and the following disclaimer. */ /* Redistributions in binary form must reproduce the above copyright */ /* notice, this list of conditions and the following disclaimer in */ /* the documentation and/or other materials provided with the distribution. */ /* */ /* Neither the name of Rexx Language Association nor the names */ /* of its contributors may be used to endorse or promote products */ /* derived from this software without specific prior written permission. */ /* */ /* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS */ /* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT */ /* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS */ /* FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT */ /* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, */ /* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED */ /* TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, */ /* OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY */ /* OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING */ /* NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS */ /* SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ /* */ /*----------------------------------------------------------------------------*/ /******************************************************************************/ /* Object REXX Kernel */ /* */ /* Main language parser transient class. */ /* */ /******************************************************************************/ #include #include "RexxCore.h" #include "LanguageParser.hpp" #include "ProgramSource.hpp" #include "MethodClass.hpp" #include "RoutineClass.hpp" #include "PackageClass.hpp" #include "RexxCode.hpp" #include "StringTableClass.hpp" #include "ClassDirective.hpp" #include "StackFrameClass.hpp" #include "ActivationFrame.hpp" #include "RexxActivation.hpp" #include "RexxLocalVariables.hpp" #include "SelectInstruction.hpp" #include "ElseInstruction.hpp" #include "EndIf.hpp" #include "EndInstruction.hpp" #include "DoInstruction.hpp" #include "OtherwiseInstruction.hpp" #include "CallInstruction.hpp" #include "ExpressionFunction.hpp" #include "ExpressionVariable.hpp" #include "ExpressionCompoundVariable.hpp" #include "ExpressionStem.hpp" #include "ExpressionDotVariable.hpp" #include "SpecialDotVariable.hpp" #include "ExpressionMessage.hpp" #include "ExpressionOperator.hpp" #include "ExpressionLogical.hpp" #include "ExpressionList.hpp" #include "RexxInternalApis.h" #include "SystemInterpreter.hpp" #include "TraceSetting.hpp" #include "ExpressionQualifiedFunction.hpp" #include "ExpressionClassResolver.hpp" #include "VariableReferenceOp.hpp" const char *ENCODED_NEEDLE = "/**/@REXX@"; // from ProgramMetaData.cpp /** * Static method for creating a new MethodClass instance. * * @param name The name given to the method and package. * @param source The code source as an array of strings. * @param sourceContext * A parent source context that this method will inherit * a package environment from. * * @return An executable method object. */ MethodClass *LanguageParser::createMethod(RexxString *name, ArrayClass *source, PackageClass *sourceContext) { // check whether the data is representing an encoded compiled Rexx program (rexxc with the '/e' switch), // i.e. the second entry consists of the string "/**/@REXX@" only; if (source->items()>1 && source->get(2)->stringValue()->strCompare(ENCODED_NEEDLE)) { Protected strSource=source->toString(GlobalNames::LINES, GlobalNames::LINEFEED); // use single LF for concatenation Protected program_buffer = new_buffer(strSource->getStringData(), strSource->getLength()); // try to restore a compiled and encoded program first Protected method = MethodClass::restore(name, program_buffer); if (method != (MethodClass *)OREF_NULL) { return method; } } // create the appropriate array source, then the parser, then generate the // code. Protected programSource = new ArrayProgramSource(source); Protected parser = new LanguageParser(name, programSource); return parser->generateMethod(sourceContext); } /** * Static method for creating a new MethodClass instance. * * @param name The name given to the method and package. * @param source The code source as a buffer * @param sourceContext * A parent source context that this method will inherit * a package environment from. * * @return An executable method object. */ MethodClass *LanguageParser::createMethod(RexxString *name, BufferClass *source) { // create the appropriate array source, then the parser, then generate the // code. Protected programSource = new BufferProgramSource(source); Protected parser = new LanguageParser(name, programSource); return parser->generateMethod(); } /** * Static method for creating a new MethodClass instance from a * file. * * @param name The filename to read the method source from. * @param sourceContext * A parent source context that this method will inherit * a package environment from. * * @return An executable method object. */ MethodClass *LanguageParser::createMethod(RexxString *name, PackageClass *sourceContext) { // load the file into a buffer Protected program_buffer = FileProgramSource::readProgram(name->getStringData()); // if this failed, report an error now. if (program_buffer == (BufferClass *)OREF_NULL) { reportException(Error_Program_unreadable_name, name); } // try to restore a flattened program first Protected method = MethodClass::restore(name, program_buffer); if (method != (MethodClass *)OREF_NULL) { // method->setPackageObject(sourceContext); // method->getPackage()->addPackage(sourceContext); // method->getPackage()->inheritPackageContext(sourceContext); // method->getPackage()->install(); return method; } // create the appropriate program source, then the parser, then generate the // code. Protected programSource = new BufferProgramSource(program_buffer); Protected parser = new LanguageParser(name, programSource); return parser->generateMethod(sourceContext); } /** * Static method for creating a new RoutineClass instance. * * @param name The name given to the routine and package. * @param source The code source as an array of strings. * @param sourceContext * A parent source context that this method will inherit * a package environment from. * * @return An executable method object. */ RoutineClass *LanguageParser::createRoutine(RexxString *name, ArrayClass *source, PackageClass *sourceContext) { // check whether the data is representing an encoded compiled Rexx program (rexxc with the '/e' switch), // i.e. the second entry consists of the string "/**/@REXX@" only; if (source->items()>1 && source->get(2)->stringValue()->strCompare(ENCODED_NEEDLE)) { Protected strSource=source->toString(GlobalNames::LINES, GlobalNames::LINEFEED); // use single LF for concatenation Protected program_buffer = new_buffer(strSource->getStringData(), strSource->getLength()); // try to restore a compiled and encoded program first Protected routine = RoutineClass::restore(name, program_buffer); if (routine != (RoutineClass *)OREF_NULL) { return routine; } } // create the appropriate array source, then the parser, then generate the // code. Protected programSource = new ArrayProgramSource(source); Protected parser = new LanguageParser(name, programSource); return parser->generateRoutine(sourceContext); } /** * Static method for creating a new RoutineClass instance from a * file. * * @param name The filename to read the routine source from. * @param sourceContext * A parent source context that this routine will inherit * a package environment from. * * @return An executable routine object. */ RoutineClass* LanguageParser::createRoutine(RexxString *name, PackageClass *sourceContext) { // load the file into a buffer Protected program_buffer = FileProgramSource::readProgram(name->getStringData()); // if this failed, report an error now. if (program_buffer == (BufferClass *)OREF_NULL) { reportException(Error_Program_unreadable_name, name); } // try to restore a flattened program first Protected routine = RoutineClass::restore(name, program_buffer); if (routine != (RoutineClass *)OREF_NULL) { return routine; } // process this from the source return createRoutine(name, program_buffer, sourceContext); } /** * Static method for creating a new RoutineClass instance. * * @param name The name given to the routine and package. * @param source The code source as a buffer * @param sourceContext * A parent source context that this method will inherit * a package environment from. * * @return An executable method object. */ RoutineClass* LanguageParser::createRoutine(RexxString *name, BufferClass *source, PackageClass *sourceContext) { // try to restore a flattened program first Protected routine = RoutineClass::restore(name, source); if (routine != (RoutineClass *)OREF_NULL) { return routine; } // create the appropriate array source, then the parser, then generate the // code. Protected programSource = new BufferProgramSource(source); Protected parser = new LanguageParser(name, programSource); return parser->generateRoutine(sourceContext); } /** * Static method for creating a new RoutineClass instance as a * top-level program (no install step is run) * * @param name The name given to the routine and package. * @param source The buffer containing the source. * * @return An executable method object. */ RoutineClass* LanguageParser::createProgram(RexxString *name, BufferClass *source) { // try to restore a flattened program first Protected routine = RoutineClass::restore(name, source); if (routine != (RoutineClass *)OREF_NULL) { return routine; } // create the appropriate array source, then the parser, then generate the // code. Protected programSource = new BufferProgramSource(source); Protected parser = new LanguageParser(name, programSource); return parser->generateProgram(); } /** * Static method for creating a new RoutineClass instance as a * top-level program (no install step is run) * * @param name The name given to the routine and package. * @param source The buffer containing the source. * * @return An executable method object. */ RoutineClass* LanguageParser::createProgram(RexxString *name, ArrayClass *source, PackageClass *sourceContext) { // check whether the data is representing an encoded compiled Rexx program (rexxc with the '/e' switch), // i.e. the second entry consists of the string "/**/@REXX@" only; if (source->items()>1 && source->get(2)->stringValue()->strCompare(ENCODED_NEEDLE)) { Protected strSource=source->toString(GlobalNames::LINES, GlobalNames::LINEFEED); // use single LF for concatenation Protected program_buffer = new_buffer(strSource->getStringData(), strSource->getLength()); // try to restore a compiled and encoded program first Protected routine = RoutineClass::restore(name, program_buffer); if (routine != (RoutineClass *)OREF_NULL) { return routine; } } // create the appropriate array source, then the parser, then generate the // code. Protected programSource = new ArrayProgramSource(source); Protected parser = new LanguageParser(name, programSource); return parser->generateProgram(sourceContext); } /** * Static method for creating a new RoutineClass instance as a * top-level program (no install step is run) * * @param name The name given to the routine and package. * @param source The buffer containing the source. * * @return An executable method object. */ RoutineClass* LanguageParser::createProgram(RexxString *name) { // create the appropriate program source, then the parser, then generate the // code. Protected programSource = new FileProgramSource(name); Protected parser = new LanguageParser(name, programSource); return parser->generateProgram(); } /** * Retrieve a routine object from a file. This will first attempt * to restore a previously translated image, then will try to * translate the source if that fails. * * @param filename The target file name. * * @return A resulting Routine object, if possible. */ RoutineClass* LanguageParser::createProgramFromFile(RexxString *filename) { // load the file into a buffer Protected program_buffer = FileProgramSource::readProgram(filename->getStringData()); // if this failed, report an error now. if (program_buffer == (BufferClass *)OREF_NULL) { reportException(Error_Program_unreadable_name, filename); } // try to restore a flattened program first Protected routine = RoutineClass::restore(filename, program_buffer); if (routine != (RoutineClass *)OREF_NULL) { return routine; } // process this from the source return createProgram(filename, program_buffer); } /** * Retrieve a package object from a file. This will first * attempt to restore a previously translated image, then will * try to translate the source if that fails. * * @param filename The target file name. * * @return A resulting Routine object, if possible. */ PackageClass *LanguageParser::createPackage(RexxString *name, ArrayClass *source, PackageClass *sourceContext) { // we just load this as a program object and return the associated package return createProgram(name, source, sourceContext)->getPackage(); } /** * Retrieve a package object from a file. This will first * attempt to restore a previously translated image, then will * try to translate the source if that fails. * * @param filename The target file name. * * @return A resulting Routine object, if possible. */ PackageClass *LanguageParser::createPackage(RexxString *name, BufferClass *source) { // we just load this as a program object and return the associated package return createProgram(name, source)->getPackage(); } /** * Create a package object from an array source. * * @param filename The package name * * @return A resulting Package object, if possible. */ PackageClass *LanguageParser::createPackage(RexxString *filename) { // we just load this as a program object and return the associated package return createProgramFromFile(filename)->getPackage(); } /** * Translate a single string value for an interpret * instruction. * * @param interpretString * The string to interpret. * @param labels The labels from the interpret context. * @param lineNumber The line number of the interpret location. * * @return The interpreted code. */ RexxCode *LanguageParser::translateInterpret(RexxString *interpretString, PackageClass *sourceContext, StringTable *labels, size_t lineNumber) { // create the appropriate array source, then the parser, then generate the // code. ProgramSource *programSource = new ArrayProgramSource(new_array(interpretString), lineNumber); // the package for the interpret will inherit the program name from the parent context. Protected parser = new LanguageParser(sourceContext->getProgramName(), programSource); return parser->translateInterpret(sourceContext, labels); } /** * Allocate a new LanguageParser item * * @param size The base object size. * * @return The storage for creating a LanguageParser. */ void *LanguageParser::operator new(size_t size) { return new_object(size, T_LanguageParser); } /** * Construct a program source object. * * @param p The source package we're parsing code for. * @param s The provider for the actual program source. */ LanguageParser::LanguageParser(RexxString *n, ProgramSource *s) { // at this point, we just save the link back to the // package and source objects. We hold off creating // more objects until we start parsing. name = n; source = s; } /** * Perform garbage collection on a live object. * * @param liveMark The current live mark. */ void LanguageParser::live(size_t liveMark) { // because of the way garbage collection works, it is a good // idea to mark these first because they are chained. memory_mark(mainSection); memory_mark(firstInstruction); memory_mark(lastInstruction); memory_mark(currentInstruction); memory_mark(package); memory_mark(source); memory_mark(name); memory_mark(clause); memory_mark(holdStack); memory_mark(variables); memory_mark(constantVariables); memory_mark(literals); memory_mark(dotVariables); memory_mark(labels); memory_mark(strings); memory_mark(guardVariables); memory_mark(exposedVariables); memory_mark(localVariables); memory_mark(control); memory_mark(terms); memory_mark(subTerms); memory_mark(operators); memory_mark(calls); memory_mark(activeClass); memory_mark(classes); memory_mark(unattachedMethods); memory_mark(classDependencies); memory_mark(routines); memory_mark(publicRoutines); memory_mark(requires); memory_mark(libraries); memory_mark(resources); } /** * Perform generalized live marking on an object. This is * used when mark-and-sweep processing is needed for purposes * other than garbage collection. * * @param reason The reason for the marking call. */ void LanguageParser::liveGeneral(MarkReason reason) { // because of the way garbage collection works, it is a good // idea to mark these first because they are chained. memory_mark_general(mainSection); memory_mark_general(firstInstruction); memory_mark_general(lastInstruction); memory_mark_general(currentInstruction); memory_mark_general(package); memory_mark_general(source); memory_mark_general(name); memory_mark_general(clause); memory_mark_general(holdStack); memory_mark_general(variables); memory_mark_general(constantVariables); memory_mark_general(literals); memory_mark_general(dotVariables); memory_mark_general(labels); memory_mark_general(strings); memory_mark_general(guardVariables); memory_mark_general(exposedVariables); memory_mark_general(localVariables); memory_mark_general(control); memory_mark_general(terms); memory_mark_general(subTerms); memory_mark_general(operators); memory_mark_general(calls); memory_mark_general(activeClass); memory_mark_general(classes); memory_mark_general(unattachedMethods); memory_mark_general(classDependencies); memory_mark_general(routines); memory_mark_general(publicRoutines); memory_mark_general(requires); memory_mark_general(libraries); memory_mark_general(resources); } /** * Generate a method object from a source collection. * * @param sourceContext * An optional source context used to add additional visibility to * dynamically generated methods. * * @return A method object represented by the leading code block * of the source. Ideally, this code should not contain directives, * but since this was allowed in the past, we need to continue * to allow this. */ MethodClass *LanguageParser::generateMethod(PackageClass *sourceContext) { // initialize, and compile all of the source. compileSource(); // get the main section of the source package and make a method // object from it. This is the package "main" executable. package->mainExecutable = new MethodClass(name, mainSection); // since we've explicitly made this a method, there is no // longer an init code section to this package. package->initCode = OREF_NULL; // if we have a source context, then we need to inherit this // context before doing the install so that anything from the parent // context is visible during the install processing. package->inheritPackageContext(sourceContext); // force the package to resolve classes/libraries now. installPackage(); // return the main executable. return (MethodClass *)package->mainExecutable; } /** * Generate a routine object from a source collection. * * @param sourceContext * An optional source context used to add additional visibility to * dynamically generated methods. * * @return A routine object represented by the leading code * block of the source. Ideally, this code should not * contain directives, but since this was allowed in the * past, we need to continue to allow this. */ RoutineClass *LanguageParser::generateRoutine(PackageClass *sourceContext) { // initialize, and compile all of the source. compileSource(); // get the main section of the source package and make a method // object from it. This is the package "main" executable. package->mainExecutable = new RoutineClass(name, mainSection); // since we've explicitly made this a method, there is no // longer an init code section to this package. package->initCode = OREF_NULL; // if we have a source context, then we need to inherit this // context before doing the install so that anything from the parent // context is visible during the install processing. package->inheritPackageContext(sourceContext); // force the package to resolve classes/libraries now. installPackage(); // return the main executable. return (RoutineClass *)package->mainExecutable; } /** * Generate a routine object as a top-level program. * * @return A routine object represented by the leading code * block of the source. This will not have an installs * performed at this time. */ RoutineClass *LanguageParser::generateProgram(PackageClass *sourceContext) { // initialize, and compile all of the source. compileSource(); // override of program/package setting, if enabled PackageClass::overridePackageSettings(package); // get the main section of the source package and make a method // object from it. This is the package "main" executable. package->mainExecutable = new RoutineClass(name, mainSection); // this has code marked as the init code that will run the first // time something on this is called. We do not do any installation // at this time. package->initCode = mainSection; // if we have a source context, then we need to inherit this // context before doing the install so that anything from the parent // context is visible during the install processing. package->inheritPackageContext(sourceContext); // return the main executable. return (RoutineClass *)package->mainExecutable; } /** * Translate an interpret string. * * @param contextlabels * The labels for the interpreting source program. * * @return A RexxCode object resulting from the compilation of * this interpret line. */ RexxCode *LanguageParser::translateInterpret(PackageClass *sourceContext, StringTable *contextLabels) { // to translate this, we use the labels from the parent context. labels = contextLabels; // turn on the interpret restrictions setInterpret(); // initialize, and compile all of the source. compileSource(); // if we have a source context, then we need to inherit this // context before doing the install so that anything from the parent // context is visible during the install processing. package->inheritPackageContext(sourceContext); // the package ends up with neither an init section or a main executable. // we just return the main code section // return the main executable. return mainSection; } /** * Load a package and return the source object for the package. * We create the main executable as a Routine object, but also * leave the initialization code in place for the package loader * to use. * * @return A package object representing the package. */ PackageClass *LanguageParser::generatePackage(PackageClass *sourceContext) { // initialize, and compile all of the source. compileSource(); // get the main section of the source package and make a method // object from it. This is the package "main" executable. package->mainExecutable = new RoutineClass(name, mainSection); // The main section is also set into the package init code to be // run as part of package loading. package->initCode = mainSection; // if we have a source context, then we need to inherit this // context before doing the install so that anything from the parent // context is visible during the install processing. package->inheritPackageContext(sourceContext); // return the package. return package; } /** * Compile our configured source into executable code. */ void LanguageParser::compileSource() { // initialize the global environment for parsing this source // into an executable. initializeForParsing(); // now translate the code translate(); } /** * Initialize the parser before starting the parse operation. * NOTE: This is a transient object, which will never be stored * in the oldspace, so we don't need to use OrefSet. */ void LanguageParser::initializeForParsing() { // create a package object that we'll be filling in. package = new PackageClass(name, source); // do setup, which also initializes the program source. package->setup(); // get the count of lines lineCount = source->getLineCount(); // and the lineNumber we should start reading from. For interpret, // this will be one line before the interpret instruction. This allows // all syntax errors to be reported on the correct line. lineNumber = source->getFirstLine(); // position at the start of that line position(lineNumber, 0); // handy stack for temporary values...this is a push through holdStack = new (HOLDSIZE) PushThroughStack(HOLDSIZE); // general parsing control setups control = new_queue(); // our stack of control instructions terms = new_queue(); // expression term stack subTerms = new_queue(); // temporary stack for holding lists of terms operators = new_queue(); // the operator queue literals = new_string_table(); // table of literal values dotVariables = new_string_table(); // table of dot variables // we have three special dot variables that are given special treatment. Go ahead and // create the retriever objects now dotVariables->put(new SpecialDotVariable(GlobalNames::DOTNIL, TheNilObject), GlobalNames::DOTNIL); dotVariables->put(new SpecialDotVariable(GlobalNames::DOTTRUE, TheTrueObject), GlobalNames::DOTTRUE); dotVariables->put(new SpecialDotVariable(GlobalNames::DOTFALSE, TheFalseObject), GlobalNames::DOTFALSE); // during an image build, we have a global string table. If this is // available now, use it. strings = memoryObject.getGlobalStrings(); if (strings == OREF_NULL) { // no global string table, use a local copy strings = new_string_table(); } // create the singleton clause object for parsing clause = new RexxClause(); } /** * Mark the package that it requires at least the given * language level to execute. * * @param l The new language level. */ void LanguageParser::requireLanguageLevel(LanguageLevel l) { LanguageLevel oldLevel = package->getLanguageLevel(); if (l > oldLevel) { package->setLanguageLevel(l); } } /** * Test if this version of the interpreter can handle * executing a program created with a given language level. * * @param l The target language level. * * @return True if the given level is within the range supported * by this interpreter version. */ bool LanguageParser::canExecute(LanguageLevel l) { return l >= MinimumLanguageLevel && l <= MaximumLanguageLevel; } /** * Initialize the global tables used for keeping track of * directive information. */ void LanguageParser::initializeForDirectives() { routines = new_string_table(); publicRoutines = new_string_table(); classDependencies = new_string_table(); requires = new_array(); libraries = new_array(); classes = new_array(); activeClass = OREF_NULL; unattachedMethods = new_string_table(); resources = new_string_table(); } /** * Have the generated package process the package * installation step at completion of parsing. */ void LanguageParser::installPackage() { // override of program/package setting, if enabled PackageClass::overridePackageSettings(package); package->install(); } /** * Create a stack frame for this parsing context. * * @return a stack frame instance for error reporting */ StackFrameClass *LanguageParser::createStackFrame() { // construct the traceback line before we allocate the stack frame object. // calling this in the constructor argument list can cause the stack frame instance // to be inadvertently reclaimed if a GC is triggered while evaluating the constructor // arguments. RexxString *traceback = package->traceBack(OREF_NULL, clauseLocation, 0, true); ProtectedObject p(traceback); return new StackFrameClass(StackFrameClass::FRAME_COMPILE, package->programName, OREF_NULL, OREF_NULL, OREF_NULL, traceback, clauseLocation.getLineNumber(), 0, OREF_NULL); } /** * Validate that a given token is a variable (i.e., a symbol * that does not begin with a digit). * * @param token The target token. */ void LanguageParser::needVariable(RexxToken *token) { // this must be a variable token. if (!token->isVariable()) { // the error message depends on whether this begins with a dot // or is a numeric value if (token->value()->getChar(0) == '.') { syntaxError(Error_Invalid_variable_period, token); } else { syntaxError(Error_Invalid_variable_number, token); } } } /** * Validate that a token is a variable or the literal * dot (e.g. the tokens allowed in a parsing template). * * @param token The target token. */ void LanguageParser::needVariableOrDotSymbol(RexxToken *token) { // check if the correct types if (!token->isVariable() && !token->isDotSymbol()) { syntaxError(Error_Invalid_variable_number, token); } } /** * Advance the current position to the next line. */ void LanguageParser::nextLine() { if (clause != OREF_NULL) // do we have a clause object active? { // record the current position in the clause...this marks the location end. clause->setEnd(lineNumber, lineOffset); } // move to the start of the next line position(lineNumber + 1, 0); } /** * Advance the current position to the next line, but only if we * are not already positioned at the first character. */ void LanguageParser::conditionalNextLine() { // if scanning of the previous clause terminated at the end of the // line, the position has already been advanced to the next line and // we'll be sitting at the first character if (lineOffset != 0) { nextLine(); } } /** * Check the current line against the end marker to see * if this is a match. * * @param marker The marker string to check. * * @return true if this line begins with the marker, false otherwise. */ bool LanguageParser::checkMarker(RexxString *marker) { size_t checkLength = marker->getLength(); // quick return if this is too short if (checkLength > currentLength) { return false; } // do a strict compare with the beginning of the line. we only trigger on // the line beginning with the string, anything else is ignored. return memcmp(marker->getStringData(), current, checkLength) == 0; } /** * Extract the current line as a string object. * * @return The current line as a string object. */ RexxString *LanguageParser::getStringLine() { return new_string(current, currentLength); } /** * Move the current scan position to a new source location. * * @param line The target line number. * @param offset The offset within the new line. */ void LanguageParser::position(size_t line, size_t offset) { lineNumber = line; // set the new position information lineOffset = offset; // retrieve the line specifics from the source object. If this // is out of bounds, current will be nulled out. source->getLine(line, current, currentLength); } /** * Extract a clause from the source and return as a clause * object. The clause object contains a list of all of the * tokens contained within the clause and is used by * the parser to determine the type of instruction and * create the instruction parse tree. * * @return true if we managed to get a clause, false otherwise. */ bool LanguageParser::nextClause() { SourceLocation location; // location of the clause SourceLocation tokenLocation; // location of each token RexxToken *token = OREF_NULL; // if reclamed is true, we have previously given up on processing a clause, so // the tokens are already available. This is not typically the case. if (!flags.test(reclaimed)) { // we keep one clause object, just reset the tokens in the clause. clause->newClause(); // scan over all null clauses. These could be blank lines, comments, // semi-colons, etc. Those are not interesting. for (;;) { // record the start position clause->setStart(lineNumber, lineOffset); // get the next source token. White space tokens are ignored in this context, // as we're looking for the beginning of a real clause. token = sourceNextToken(OREF_NULL); // OREF_NULL indicates we've hit the end of the source. Mark us as // finished and return if (token->isEndOfFile()) { flags.set(noClause); return false; } // we have a token, but is it an end of clause marker (explicit or implicit // semicolon). If not, we've got a real clause to process. if (!token->isEndOfClause()) { break; } // reset the clause and start again. clause->newClause(); } // get the clause start position from the first token, and // make a copy for the start tokenLocation = token->getLocation(); location = tokenLocation; // record this in the clause for potential error reporting. clause->setLocation(location); // now consume tokens until we find an end of clause marker. for (;;) { // get the next token. Blanks can now be significant inside // a clause. token = sourceNextToken(token); // and save the position again. tokenLocation = token->getLocation(); // and quit once we've hit an end-of-clause (note, we'll never // see a null return here because there will be an end of clause // implied by the end of line. We only see the null case when // we're scanning for a new clause. if (token->isEndOfClause()) { break; } } // the location of the last scanned token is the clause end position. location.setEnd(tokenLocation); // and set this in the class to give it the full bounds. clause->setLocation(location); } // no reclaimed clause now, regardless of how we got here. flags.reset(reclaimed); // always set the error information for the clause being processed. clauseLocation = clause->getLocation(); // we have a clause return true; } /** * Translate a source object into executable code. This translates * the main code section and stores it in the mainSection * field. If permitted (i.e, not an interpret), this will * also process any additional directives and associated * code sections. */ void LanguageParser::translate() { // create a stack frame so errors can display the parsing location. CompileActivationFrame frame(ActivityManager::currentActivity, this); // set up the package global defaults package->packageSettings.setDefault(); // go translate the lead block. We will figure out what type of // object this gets turned into later. mainSection = translateBlock(); // we might have directives to process, which adds additional stuff // to the package. if (!atEnd()) { // we store a lot of stuff that we only need if there are directives. // set up to handle this now. initializeForDirectives(); // no active class definition activeClass = OREF_NULL; // We only allow directives when translating ... interpret need not apply if (flags.test(noDirectives)) { // step to the next clause to report the error nextClause(); syntaxError(Error_Translation_directive_interpret); } // now loop until we hit the end of the source processing directives. while (!atEnd()) { nextDirective(); } // resolve any class dependencies resolveDependencies(); } } /** * Initialize all of the fields that need to start clean for a new * translation section. */ void LanguageParser::initializeForTranslation() { // initialize the parsing environment. firstInstruction = OREF_NULL; lastInstruction = OREF_NULL; // get a list of all calls that might need resolution calls = new_array(); // a table of variables...starting with the special variables we allocated space for. variables = (StringTable *)TheCommonRetrievers->copy(); // restart the variable index variableIndex = RexxLocalVariables::FIRST_VARIABLE_INDEX; // do we have labels from an interpret? // only create a new set if we're not reusing. if (labels == OREF_NULL) { labels = new_string_table(); } // until we need guard variables, we don't need the table guardVariables = OREF_NULL; // and we reset the exposed/local variables for each code section exposedVariables = OREF_NULL; localVariables = OREF_NULL; // clear the stack accounting fields maxStack = 0; currentStack = 0; // we're not at the end yet flags.reset(noClause); } /** * Translate a block of REXX code (delimited by possible * directive instructions * * @return A RexxCode object for this block. */ RexxCode *LanguageParser::translateBlock() { // reset for a fresh translation initializeForTranslation(); // add a dummy instruction at the front. All other instructions get chained off of this. RexxInstruction *instruction = new RexxInstruction(OREF_NULL, KEYWORD_FIRST); // this is the bottom of the control stack, and also the // first clause of the code stream. firstInstruction = instruction; lastInstruction = instruction; pushDo(instruction); // get a location for this block of code from the first and last instructions SourceLocation blockLocation; // save the block start position blockLocation.setStart(lineNumber == 0 ? 1 : lineNumber, lineOffset); // time to start actual parsing. Continue until we reach the end nextClause(); for (;;) { // start with no instruction instruction = OREF_NULL; // At this point, we want to consume any label clauses, since they are // not real instructions. while (!noClauseAvailable()) { // resolve this clause into an instruction instruction = nextInstruction(); // if nothing is returned, this must be a directive, which terminates // parsing of this block. if (instruction == OREF_NULL) { break; } // not a label, break out of the loop if (!instruction->isType(KEYWORD_LABEL)) { break; } // labels are not allowed within DO/LOOP/IF/SELECT groups InstructionKeyword type = topDo()->getType(); // We are // - inside a DO, if top isControl, i. e. KEYWORD_LOOP_* or _DO_* // - inside a SELECT if top is KEYWORD_SELECT / SELECT_CASE / WHENTHEN / ENDWHEN / OTHERWISE // - inside an IF if top is IFTHEN or KEYWORD_ELSE // If ENDTHEN is at the top this is somewhat special. In this case // this label here may be preceding an ELSE (so we are inside the IF) // or something else (meaning we're just outside the IF block). // We handle this in the KEYWORD_ELSE path below. bool withinIf = type == KEYWORD_IFTHEN || type == KEYWORD_ELSE; bool withinSelect = type == KEYWORD_SELECT || type == KEYWORD_SELECT_CASE || type == KEYWORD_WHENTHEN || type == KEYWORD_ENDWHEN || type == KEYWORD_OTHERWISE; if (topDo()->isControl() || withinIf || withinSelect) { // get our label name resetPosition(1); // firstToken() won't do .. RexxToken *token = nextReal(); syntaxError( withinIf ? Error_Unexpected_label_if : (withinSelect ? Error_Unexpected_label_select : Error_Unexpected_label_do), token); } // append the label and try again addClause(instruction); nextClause(); // need to zero this out in case we break the loop for an end of file instruction = OREF_NULL; } // ok, have we hit the end of the file or the end of the block? if (noClauseAvailable() || instruction == OREF_NULL) { // see what we have at the top of the control stack, we probably // have some cleanup to do. InstructionKeyword controltype = topDoType(); // handle any then or when terminators while (controltype == KEYWORD_ENDTHEN || controltype == KEYWORD_ENDWHEN) { // pop these from the stack and flush and pending control instructions popDo(); flushControl(OREF_NULL); controltype = topDoType(); } // In theory, at this point we should be at the first instruction. // if not, we have an unclosed block instruction, which is an error if (!topDoIsType(KEYWORD_FIRST)) { blockSyntaxError(topDo()); } // remove the top instruction from the stack, and we're done popDo(); break; } // now check if we need to adjust the control stack for this new instruction. InstructionKeyword type = instruction->getType(); // if this is not an ELSE, we might have a pending THEN to finish. if (type != KEYWORD_ELSE) { // get the type at the top of the stack. InstructionKeyword controltype = topDoType(); // we might need to pop off multiple pending thens while end of an IF or WHEN while (controltype == KEYWORD_ENDTHEN || controltype == KEYWORD_ENDWHEN) { popDo(); flushControl(OREF_NULL); controltype = topDoType(); } } // now check the actual disposition of this instruction. If it is a control type // add it immediately to the stream. Control types are IF, SELECT, or any of the // DO/LOOP types. if (instruction->isControl()) { addClause(instruction); } // if this is an ELSE, we don't add a new level, but rather flush // any pending control levels. else if (type != KEYWORD_ELSE) { flushControl(instruction); } // validate allowed instructions in a SELECT if (topDoIsType(KEYWORD_SELECT, KEYWORD_SELECT_CASE) && (type != KEYWORD_WHEN && type != KEYWORD_WHEN_CASE && type != KEYWORD_OTHERWISE && type != KEYWORD_END)) { syntaxError(Error_When_expected_whenotherwise, topDo()); } // now handle the different instructions to ensure they // are valid in their particular contexts. switch (type) { // a WHEN clause. The top of the Do stack must be a select. case KEYWORD_WHEN: case KEYWORD_WHEN_CASE: { // the top of the queue must be a SELECT instruction, but // we have TWO varieties of this. The second requires the // WHEN to be converted to a different type. RexxInstruction *second = topDo(); if (second->isType(KEYWORD_SELECT) || second->isType(KEYWORD_SELECT_CASE)) { // let the select know that another WHEN was added ((RexxInstructionSelect *)second)->addWhen((RexxInstructionIf *)instruction); } } // processing of an IF instruction, and also a WHEN (from above) case KEYWORD_IF: { // we need to finish the IF instruction. // get the next token RexxToken *token = nextReal(); RexxInstruction *second; // Did the line end with no THEN? It must be on the next // line, if (token->isEndOfClause()) { // get the next full clause, which should start with // the THEN keyword. // did we hit the end of file?, this is an error if (!nextClause()) { // select appropriate error message: either for the IF here, // or for the WHEN / WHEN_CASE we fell through from above syntaxError(type == KEYWORD_IF ? Error_Then_expected_if : Error_Then_expected_when, instruction); } // now check the next token and ensure it is a THEN keyword. token = nextReal(); // Not a THEN keyword? This is an error if (token->keyword() != KEYWORD_THEN) { // select appropriate error message: either for the IF here, // or for the WHEN / WHEN_CASE we fell through from above syntaxError(type == KEYWORD_IF ? Error_Then_expected_if : Error_Then_expected_when, instruction); } // create a new then clause attached to the IF second = thenNew(token, (RexxInstructionIf *)instruction); // now get the next token.. and ensure is something after the THEN token = nextReal(); // if this is a clause end (e.g, end of line), parse off a clause // to make sure there is something there if (token->isEndOfClause()) { if (!nextClause()) { syntaxError(Error_Incomplete_do_then, instruction); } } else { // step back a token and trim the THEN off of the clause. previousToken(); trimClause(); } } // we have THEN on the same line as the IF else { // attach a THEN to the IF second = thenNew(token, (RexxInstructionIf *)instruction); // there might not be anything after the THEN, so we have to check token = nextReal(); // if the THEN was the last thing, ensure we have something on the // next line if (token->isEndOfClause()) { // we must have a clause, else there is an error if (!nextClause()) { syntaxError(Error_Incomplete_do_then, instruction); } } else { // ok, back up one token and trim the clause to remove the THEN previousToken(); trimClause(); } } // add this to the instruction list and also add this to the // DO stack in case there is an ELSE to process. addClause(second); pushDo(second); // back to the top to get a new instruction continue; } // we have an ELSE instruction. Need to verify this is in a correct context. case KEYWORD_ELSE: { // ok, the top instruction is the key. It must be the // tail end of a THEN instruction that we can attach to. RexxInstruction *second = topDo(); if (!second->isType(KEYWORD_ENDTHEN)) { syntaxError(Error_Unexpected_then_else); } // no label is allowed immediately before the ELSE keyword if (lastInstruction->isType(KEYWORD_LABEL)) { // unfortunately the label name isn't readily available // probably a rare error .. just use "ELSE" as message insert syntaxError(Error_Unexpected_label_if, new_string("ELSE")); } // ok, add the ELSE to the instruction list, pop the // THEN off of the control stack, push the ELSE on to the stack, // and hook them together. The ELSE will need to be completed // once the next instruction has been parsed off. addClause(instruction); second = popDo(); // this is the ELSE pushDo(instruction); // join the THEN and ELSE together ((RexxInstructionElse *)instruction)->setParent((RexxInstructionEndIf *)second); ((RexxInstructionEndIf *)second)->setEndInstruction((RexxInstructionEndIf *)instruction); // check for a dangling ELSE now. RexxToken *token = nextReal(); // if the next token is the end of the line (or potentially, a semicolon) // we're not at the end. if (token->isEndOfClause()) { // dangling ELSE if we can't get another clause. if (!nextClause()) { syntaxError(Error_Incomplete_do_else, instruction); } } // ELSE followed by an instruction on the same line. We need to // push the token we grabbed back on to the clause and trim off the // ELSE part we've already process. else { previousToken(); trimClause(); } // continue parsing instructions. continue; } // found the start of an OTHERWISE group. We need to verify that // we're really working on a SELECT. case KEYWORD_OTHERWISE: { // we must have a SELECT at the top of the control stack RexxInstruction *second = topDo(); if (!second->isType(KEYWORD_SELECT) && !second->isType(KEYWORD_SELECT_CASE)) { syntaxError(Error_Unexpected_when_otherwise); } // hook the otherwise up to the SELECT and push the otherwise // on to the top of the stack until we find an END. ((RexxInstructionSelect *)second)->setOtherwise((RexxInstructionOtherwise *)instruction); pushDo(instruction); // we could have the OTHERWISE on the same line as its following instruction, so // we need to trim the instruction RexxToken *token = nextReal(); if (!token->isEndOfClause()) { previousToken(); trimClause(); // we have a clause, go back around and process this continue; } // on the next line, fall through to the bottom of the loop // to get the next clause. break; } // An END instruction. This could be the closure for a DO, LOOP, or SELECT. // its matchup should be on the top of the control stack. case KEYWORD_END: { // ok, pop the top instruction. If this is the // correct type, we're finished with this. If not the // correct type, we have an error. RexxInstruction *second = popDo(); type = second->getType(); // that the top of the stack is an acceptable block-type instruction. // This includes all of the DO/LOOP variants, both Select types, and Otherwise if (!second->isBlock()) { // we have a couple of specific errors based on what sort of instruction is // on the top. if (type == KEYWORD_ELSE) { syntaxError(Error_Unexpected_end_else); } else if (type == KEYWORD_IFTHEN || type == KEYWORD_WHENTHEN) { syntaxError(Error_Unexpected_end_then); } else { syntaxError(Error_Unexpected_end_nodo); } } // ok, we have something good, now there is some additional // processing required for specific block instructions // if the top of the stack is an OTHERWISE, then the // END is really closing the SELECT. Replace the Otherwise // with the Select. if (type == KEYWORD_OTHERWISE) { second = popDo(); } // Now do the appropriate closure action based on the instruction type. if (type == KEYWORD_SELECT || type == KEYWORD_SELECT_CASE) { ((RexxInstructionSelect *)second)->matchEnd((RexxInstructionEnd *)instruction, this); } else /* must be a DO block */ { ((RexxBlockInstruction *)second)->matchEnd((RexxInstructionEnd *)instruction, this); } // We've just completed a large block instruction. It is possible that // this was part of something like "if a then do;..end", in which case, the // conditional instruction is still in a pending state. Flush out the control // stack to complete now that the block has been closed. flushControl(OREF_NULL); break; } // start of new DO group (also picks up LOOP instruction) // this gets pushed on to the top of the control stack until // we find an END instruction. case KEYWORD_DO: case KEYWORD_LOOP: case KEYWORD_SIMPLE_BLOCK: case KEYWORD_LOOP_FOREVER: case KEYWORD_LOOP_OVER: case KEYWORD_LOOP_OVER_UNTIL: case KEYWORD_LOOP_OVER_WHILE: case KEYWORD_LOOP_OVER_FOR: case KEYWORD_LOOP_OVER_FOR_UNTIL: case KEYWORD_LOOP_OVER_FOR_WHILE: case KEYWORD_LOOP_WITH: case KEYWORD_LOOP_WITH_UNTIL: case KEYWORD_LOOP_WITH_WHILE: case KEYWORD_LOOP_WITH_FOR: case KEYWORD_LOOP_WITH_FOR_UNTIL: case KEYWORD_LOOP_WITH_FOR_WHILE: case KEYWORD_LOOP_CONTROLLED: case KEYWORD_LOOP_CONTROLLED_UNTIL: case KEYWORD_LOOP_CONTROLLED_WHILE: case KEYWORD_LOOP_COUNT: case KEYWORD_LOOP_COUNT_UNTIL: case KEYWORD_LOOP_COUNT_WHILE: case KEYWORD_LOOP_WHILE: case KEYWORD_LOOP_UNTIL: { pushDo(instruction); break; } // new select group. Again, we push this on the start of the stack // while it awaits its associated WHEN, OTHERWISE, and END bits. case KEYWORD_SELECT: case KEYWORD_SELECT_CASE: { pushDo(instruction); break; } // all other types of instructions don't require additional processing. default: break; } // grab another clause and go around. nextClause(); } // we might have function calls or call instructions that had deferred resolution, // take care of those now resolveCalls(); // the first instruction is just a dummy we use to anchor // everything while parsing. We can unchain that now. firstInstruction = firstInstruction->nextInstruction; // if this code block does not contain labels (pretty common if // using an oo style), get rid of those too if (labels->isEmpty()) { labels = OREF_NULL; } // a code block need not have any instructions, and it may have leading or // trailing blanks. The source block started at the very beginning, now we need // find out the location of whatever terminated the block, which will be either // a directive clause that has already been parsed or actual end of the source itself. // if this block was terminated by discovering another directive, then // we set the end of this block to just before if (clause != OREF_NULL) { // get the next clause. If this clause starts at the beginning of // a line (common), then we need to use the end of the previous line as the // end location. SourceLocation nextLocation = clause->getLocation(); if (nextLocation.getOffset() == 0) { size_t previousLine = nextLocation.getLineNumber() - 1; // this might actually have stepped things back a tick if there truly // was nothing there. In that case, set the source location to zero to indicate // there was nothing there if (blockLocation.getLineNumber() > previousLine) { blockLocation.setLineNumber(0); } else { // we need this to be the end of the previous line const char *data = NULL; size_t lineLength = 0; source->getLine(previousLine, data, lineLength); blockLocation.setEnd(previousLine, lineLength); } } // we can just set the end to the beginning of the pending clause else { blockLocation.setEnd(nextLocation.getLineNumber(), nextLocation.getOffset()); } } // no active clause, so we set this to the very end of the file else { // we need this to be the end of the previous line const char *data = NULL; size_t lineLength = 0; source->getLine(source->getLineCount(), data, lineLength); blockLocation.setEnd(source->getLineCount(), lineLength); } // now create a code object that is attached to the package. // this will have all of the information needed to execute this code. RexxCode *code = new RexxCode(package, blockLocation, firstInstruction, labels, maxStack, variableIndex); // we don't automatically create the labels when we translate the block because // they might have been provided by an interpret. So always clear them out at the // end of a block. labels = OREF_NULL; // and return the code object. return code; } /** * Attempt to resolve and deferred function or call instructions. */ void LanguageParser::resolveCalls() { // ok, we have a stack of pending call/function calls to handle. // now that we've got all of the labels scanned off, we can figure out // what sort of targets these calls will resolve to. size_t count = calls->items(); for (size_t i = 1; i <= count; i++) { RexxInstruction *instruction = (RexxInstruction *)calls->get(i); // function calls are expression objects, while CALLs // are instructions. Similar, but have different // processing methods if (isOfClass(FunctionCallTerm, instruction)) { ((RexxExpressionFunction *)instruction)->resolve(labels); } else { // ok, technically, this could be either a CALL or a SIGNAL. ((RexxInstructionCallBase *)instruction)->resolve(labels); } } } /** * Translate an expression of REXX code on a directive (already * have an active clause, not translating a block * yet) * * @param token The left paren delimiter for the expression. Required for error reporting. * * @return A translated expression for this directive. */ RexxInternalObject *LanguageParser::translateConstantExpression(RexxToken *token, RexxErrorCodes error) { // reset for a fresh translation initializeForTranslation(); // since constants associated with a class are not translated in // strictly back-to-back fashion, we need to restore accumulated values maxStack = constantMaxStack; variableIndex = constantVariableIndex; if (constantVariables != OREF_NULL) { variables = constantVariables; } // parse out a subexpression, terminating on the end of clause or // a right paren (the right paren is actually the required terminator) // we get called here because we've already seen the left paren. RexxInternalObject *exp = requiredExpression(TERM_RIGHT, error); // now copy back the values we need to cache constantMaxStack = maxStack; constantVariableIndex = variableIndex; constantVariables = variables; // now verify that the terminator token was a right paren. If not, // issue an error message using the original opening token so we know // which one is an issue. if (!nextToken()->isRightParen()) { syntaxErrorAt(Error_Unmatched_parenthesis_paren, token); } // protect the expression from GC and return it. holdObject(exp); // resolve any function calls that might have been used on the constant resolveCalls(); return exp; } /** * Locate the top block instruction in the control stack. This * involves skipping over any THEN instructions waiting for * resolution. * * @return The top block instruction or OREF_NULL if none are active. */ RexxInstruction *LanguageParser::topBlockInstruction() { // iterate through the control stack until we find a real // control instruction for (size_t i = 1; i <= control->lastIndex(); i++) { RexxInstruction *inst = (RexxInstruction *)control->get(i); // We can have various THEN or ELSE or terminators for those // instructions on the stack. Keep drilling until we find a block // instruction type if (inst->isBlock()) { return inst; } } // no control instruction found return OREF_NULL; } /** * Resolve dependencies between ::CLASS directives, * rearranging the order of the directives to preserve * relative ordering wherever possible. Classes with no * dependencies in this source file will be done first, * followed by those with dependencies in the appropriate * order */ void LanguageParser::resolveDependencies() { // if we have classes, here, we need to sort out the install order // and configure the source package with this. if (!classes->isEmpty()) { // get the count of classes we need to process. size_t classCount = classes->items(); // run through the class list having each directive set up its // dependencies for (size_t i = 1; i <= classCount; i++) { ClassDirective *currentClass = (ClassDirective *)classes->get(i); currentClass->addDependencies(classDependencies); } // get an array for handling the ordering ArrayClass *classOrder = new_array(classCount); ProtectedObject p2(classOrder); // now we repeatedly scan the pending directory looking for a class // with no in-program dependencies - it's an error if there isn't one // as we build the classes we have to remove them (their names) from // pending list and from the remaining dependencies while (!classes->isEmpty()) { // this is the next one we process ClassDirective *nextInstall = OREF_NULL; // the count will update each time through. classCount = classes->items(); for (size_t i = 1; i <= classCount; i++) { // get the next directive ClassDirective *currentClass = (ClassDirective *)classes->get(i); // if this class doesn't have any additional dependencies, pick it next. if (currentClass->dependenciesResolved()) { nextInstall = currentClass; // add this to the class ordering classOrder->append(nextInstall); // remove this from the processing list classes->deleteItem(i); break; } } // if nothing was located during this pass, we must have circular dependencies // this is an error. if (nextInstall == OREF_NULL) { // directive line where we can give as the source of the error ClassDirective *errorClass = (ClassDirective *)classes->get(1); clauseLocation = errorClass->getLocation(); syntaxError(Error_Execution_cyclic, name); } // ok, now go remove these from the dependencies RexxString *className = nextInstall->getName(); // now go through the pending list telling each of the remaining classes that // they can remove this dependency from their list // the count will update each time through. classCount = classes->items(); for (size_t i = 1; i <= classCount; i++) { ClassDirective *currentClass = (ClassDirective *)classes->get(i); currentClass->removeDependency(className); } } // now add this to the package package->classes = classOrder; // this requires an install step package->setNeedsInstallation(); } // ok, now we need to fill in any additional bits needed by the package if (!requires->isEmpty()) { package->requires = requires; // this requires an install step package->setNeedsInstallation(); } if (!libraries->isEmpty()) { package->libraries = libraries; // this requires an install step package->setNeedsInstallation(); } if (!routines->isEmpty()) { package->routines = routines; } if (!publicRoutines->isEmpty()) { package->publicRoutines = publicRoutines; } if (!unattachedMethods->isEmpty()) { package->unattachedMethods = unattachedMethods; } if (!resources->isEmpty()) { package->resources = resources; } } /** * Flush an pending instructions from the control stack * for a new added instruction. * * @param instruction * The newly added instruction. */ void LanguageParser::flushControl(RexxInstruction *instruction) { // loop through the control stack for (;;) { // get the type of the instruction at the top of the control // stack. InstructionKeyword type = topDoType(); /* get the instruction type */ // is this a pending ELSE clause? */ if (type == KEYWORD_ELSE) { // pop the instruction off of the stack RexxInstruction *second = popDo(); // and create a new end marker for that instruction. second = endIfNew((RexxInstructionIf *)second); // have an instruction to add? if (instruction != OREF_NULL) { // add to the current location and don't process any additional // instructions. addClause(instruction); instruction = OREF_NULL; } // now add the else terminator behind this. addClause(second); // we can go around again on this one. } // nested IF-THEN situation? else if (type == KEYWORD_IFTHEN || type == KEYWORD_WHENTHEN) { // get the top item RexxInstruction *second = popDo(); // have an instruction to add? if (instruction != OREF_NULL) { // insert this here and null it out. addClause(instruction); instruction = OREF_NULL; } // we need a new end marker second = endIfNew((RexxInstructionIf *)second); // we add this clause behind the new one, and also add this // to the control stack as a pending instruction addClause(second); pushDo(second); // we're done with this break; } // some other type of construct. We just add the instruction to the // execution stream else { if (instruction != OREF_NULL) { addClause(instruction); } // all done flushing break; } } } /** * Test if a named variable has been previously * exposed. * * @param varName The variable name. * * @return True if the variable has been exposed, false otherwise. */ bool LanguageParser::isExposed(RexxString *varName) { // if we had an explicit EXPOSE instruction, check if this was listed if (exposedVariables != OREF_NULL) { return exposedVariables->hasIndex(varName); } // did we have a USE LOCAL instruction specified? Variable is exposed else if (localVariables != OREF_NULL) { return !localVariables->hasIndex(varName); } // neither situation, not an exposed variable return false; } /** * Perform a variable capture operation if we're * evaluating a GUARD WHEN expression. * * @param varname The name of the variable * @param retriever The associated variable retriever. */ void LanguageParser::captureGuardVariable(RexxString *varname, RexxVariableBase *retriever) { // are we collecting guard variable references (usually the case when // parsing a GUARD WHEN expression. if (capturingGuardVariables()) { // in the context of this method, we have a list of exposed variables. // if this is one of the exposed variables, then also add this to the guard // variables list so we know which variables will force a re-evaluation. if (isExposed(varname)) { // add to the guard list guardVariables->put(retriever, retriever); } } } /** * Resolve a variable name to a single common retriever object * per code block. * * @param varname The name of the target variable. * * @return A retrieve object for accessing the variable. This * will identify a variable slot in the code stack frame. */ RexxVariableBase *LanguageParser::addSimpleVariable(RexxString *varname) { // we might have this already (fairly common in most programs). If // not we cache a new one for the next time. RexxVariableBase *retriever = (RexxVariableBase *)variables->get(varname); // if not in the table yet, we need to create a new one if (retriever == OREF_NULL) { // ok, have to create a new one. // if we're in normal operation, we allocate a slot in the stack frame // and tie a variable reference to that slot. If this is an interpret, // then the variable value must be resolved dynamically. if (!isInterpret()) { variableIndex++; retriever = new RexxSimpleVariable(varname, variableIndex); } else { // a slot index of zero tells the retriever to perform a dynamic // lookup retriever = new RexxSimpleVariable(varname, 0); } // and add this to the table. variables->put(retriever, varname); } // we need to always perform the capturing test because we allow e. g. // USE LOCAL; GUARD ON/OFF WHEN v > 0 captureGuardVariable(varname, retriever); // return the variable accesser, either a new one or one pulled from the cache. return retriever; } /** * Add a stem variable instance to the execution instance. * This is like the addVariable, but the retriever * in question is for a Stem variable. * * @param stemName The name of the stem (including the period). * * @return A retriever for the specified variable. */ RexxStemVariable *LanguageParser::addStem(RexxString *stemName) { // like with normal variables, we might have this already cached. RexxStemVariable *retriever = (RexxStemVariable *)(variables->get(stemName)); if (retriever == OREF_NULL) { if (!isInterpret()) { // non-interpret uses an allocated stack frame slot variableIndex++; retriever = new RexxStemVariable(stemName, variableIndex); } else { // interpret uses dynamic lookup retriever = new RexxStemVariable(stemName, 0); } variables->put(retriever, stemName); } // we need to always perform the capturing test because we allow e. g. // USE LOCAL; GUARD ON/OFF WHEN v > 0 captureGuardVariable(stemName, retriever); // return variable accesser return retriever; } /** * Add a compound variable reference to our variable tables. * This is a two part operation...first we make sure the * stem variable is added, which uses a specific slot. Then * we use that information to create a retriever for * the specific compound variable. * * @param name The name of the compound variable. * * @return A retriever for this variable. */ RexxCompoundVariable *LanguageParser::addCompound(RexxString *name) { // we cache compound variables also...see if we've encountered // this exact name before. RexxCompoundVariable *retriever = (RexxCompoundVariable *)(variables->get(name)); if (retriever != OREF_NULL) { return retriever; } // we need to start out by scanning the name to separate the // stem and tail portions of the variable name. size_t length = name->getLength(); const char *_position = name->getStringData(); const char *start = _position; const char *end = _position + length; // we know this is a compound, so there must be at least one period. while (*_position != '.') { _position++; } // create a Rexx string version of the stem name and retrieve this // from the RexxString *stemName = new_string(start, _position - start + 1); RexxStemVariable *stemRetriever = addStem(stemName); ProtectedObject p(stemRetriever); // now split the tail piece into its component parts so that // we can optimize construction of the final tail lookup. size_t tailCount = 0; do { // we're here because we just saw a previous period. that's either the // stem variable period or the last tail element we processed. // either way, we step past it. If this period is a trailing one, // we'll add a null tail element, which is exactly what we want. _position++; start = _position; // scan for the next period while (_position < end) { if (*_position == '.') // found the next one? { break; // stop scanning now } _position++; // continue looking } // we found a tail piece. Now figure out if this is // something that cannot be a variable. Also make sure // this is part of the common string cache. RexxString *tail = commonString(new_string(start, _position - start)); if (!(tail->getLength() == 0 || (*start >= '0' && *start <= '9'))) { // this is a real variable piece. Add this to the global // variable table and push its retriever on to the term stack // for safe keeping. subTerms->push((addSimpleVariable(tail))); } else { // this is a constant string value. We can evaluate this // directly, so push it directly on to the stack. subTerms->push(tail); } // make sure we count the number of tails tailCount++; } while (_position < end); // This will pull the items off of the term stack and populate its internal // tail table for lookup processing. retriever = new (tailCount) RexxCompoundVariable(stemName, stemRetriever->getIndex(), subTerms, tailCount); // add this to the retriever table so that all references to a compound variable with the // same full name will resolved to the same retriever. We're safe just using the // variables table for this. variables->put(retriever, name); // NOTE: compound variables do get added to the guard list. return retriever; } /** * Add a variable name to the list of variables exposed * by a method. * * @param name The name of the variable. */ void LanguageParser::expose(RexxString *name ) { // create the table on the first expose if (exposedVariables == OREF_NULL) { exposedVariables = new_string_table(); } exposedVariables->put(name, name); } /** * We have a USE LOCAL specified, so turn on the auto expose. */ void LanguageParser::autoExpose() { localVariables = new_string_table(); // add the special local variables to the list localVariables->put(GlobalNames::SUPER, GlobalNames::SUPER); localVariables->put(GlobalNames::SELF, GlobalNames::SELF); localVariables->put(GlobalNames::RC, GlobalNames::RC); localVariables->put(GlobalNames::RESULT, GlobalNames::RESULT); localVariables->put(GlobalNames::SIGL, GlobalNames::SIGL); } /** * Add a variable name to the list of explicitly declared to be * local. * * @param name The name of the variable. */ void LanguageParser::localVariable(RexxString *name ) { localVariables->put(name, name); } /** * Compress all string tokens needed by a group of programs into * a single, common set of strings. It is fairly common * for a program to have duplicate versions of strings * in the form of variable names, etc. This ensures there * is just a single version of these available. * * @param string The string to add to the set. * * @return The commonized string. If this string is already * in the common table, the previously added one will be * returned. */ RexxString *LanguageParser::commonString(RexxString *string) { // check the global table first for this value. RexxString *result = (RexxString *)strings->get(string); // if not in the table, we add this new one, otherwise we // return the table version if (result != OREF_NULL) { return result; } strings->put(string, string); return string; } /** * Handle adding a variable to a program. This verifies that * the token is a valid variable, then creates a retriever that * can be used in an expression to obtain the variable value. * * @param token The token holding the variable name. * * @return A retriever object that can be used as an expression * term. */ RexxVariableBase *LanguageParser::addVariable(RexxToken *token) { // we first validate that this token represents a valid variable, needVariable(token); // then create the variable retriever object. return (RexxVariableBase *)addText(token); } /** * Handle adding a variable to a program. This verifies that * the token is a valid variable, then creates a retriever that * can be used in an expression to obtain the variable value. * * @param token The token holding the variable name. * * @return A retriever object that can be used as an expression * term. */ RexxVariableBase *LanguageParser::requiredVariable(RexxToken *token, const char *keyword) { // first verify that this is a symbol if (!token->isSymbol()) { syntaxError(Error_Symbol_expected_after_keyword, new_string(keyword)); } // now process the variable return addVariable(token); } // generate a retriever for a specific token type. Note that we // keep two caches of retrievers here. There are tokens that true // literals (strings, dot-variables, etc) and variable text tokens. The // literals can be kept in a common set across the entire package and // reused. The variables, however, contain information that is unique // to each code block within the package. These need to be kept in a // different list that is rebuilt in each code block. RexxInternalObject *LanguageParser::addText(RexxToken *token) { // these should be text type tokens that have a real value. RexxString *name = token->value(); // NOTE: We cannot check the literals table at the beginning without // knowing the type of token first. It is possible for both a literal string // and a variable name or dotvariable to have the same value, which could result in the // string constant getting used where a real variable retriever is required. // now switch on the major token class id. switch (token->type()) { // various categories of symbols. case TOKEN_SYMBOL: { // each symbol subtype requires a // different retrieval method switch (token->subtype()) { // the dummy placeholder period and symbols // that start with a digit are just literal strings. case SYMBOL_DUMMY: case SYMBOL_CONSTANT: { // we might already have processed this before. // if not, we need to examine this and find the // most appropriate form. RexxInternalObject *retriever = literals->get(name); if (retriever != OREF_NULL) { return retriever; } RexxInternalObject *value; // if this is a pure integer value within the default // digits, create an integer object if (token->isIntegerConstant()) { value = name->requestInteger(Numerics::REXXINTEGER_DIGITS); // this should not happen, given we've already validated // this, but belt and braces and all that...just // stick with the string value if it does occur. if (value == TheNilObject) { value = name; } else { // snip off the string number string // value that was created when the // integer value was created. This // is rarely used, but contributes // to the saved program size. It will // be rebuilt on demand if it is needed. name->setNumberString(OREF_NULL); } } else { // just use the string value, but also try to create and // attach the string's numeric value. value = name; name->setNumberString(value->numberString()); } // and stash the retriever value so we can resolve this if used again. literals->put(value, name); // strings and integers work directly as expression terms, so we can // just return this directly. return value; break; } // simple variable. case SYMBOL_VARIABLE: { // do the variable resolution return addSimpleVariable(name); break; } // stem variable, handled much like simple variables. case SYMBOL_STEM: { return addStem(name); break; } // compound variable...need to chop this up into its // component pieces. case SYMBOL_COMPOUND: { return addCompound(name); break; } // this is a non-numeric symbol that starts with a dot. These // are treated as environment symbols. case SYMBOL_DOTSYMBOL: { // we might already have processed this before. // if not, we need to examine this and find the // most appropriate form. Note that the constant, non-overridable // variables .nil, .true, and .false were added to the common set // up front, so we don't need to perform special checks here. RexxInternalObject *retriever = dotVariables->get(name); if (retriever != OREF_NULL) { return retriever; } // create the shorter name and add to the common set RexxString *shortName = commonString(name->extract(1, name->getLength() - 1)); // create a retriever for this using the shorter name. retriever = new RexxDotVariable(shortName); // we can add this to the literals list, since they do not // depend upon context. dotVariables->put(retriever, name); return retriever; break; } // invalid symbol subtype (should really never happen) default: { reportException(Error_Interpretation_switch, "symbol subtype", token->subtype()); break; } } break; } // just a straight literal string case TOKEN_LITERAL: { // we might already have processed this before. // if not, we need to examine this and find the // most appropriate form. RexxInternalObject *retriever = literals->get(name); if (retriever != OREF_NULL) { return retriever; } // strings are their own expression retrievers, so just add // this to the table and return it directly literals->put(name, name); return name; break; } // not a token type that can have a retriever default: { break; } } // not a token type that can have a retriever return OREF_NULL; } /** * Build a retriever for a string name. This will be * a version that uses dynamic lookup for variables rather * than using pre-assigned slots. Generally used for * dynamic lookups such as from the variable pool interface. * * @param name The variable name. * * @return A retriever object for looking up this value. */ RexxVariableBase *LanguageParser::getRetriever(RexxString *name) { // first validate that this is a symbol and get the type of symbol. switch (name->isSymbol()) { // simple variable name case STRING_NAME: return (RexxVariableBase *)new RexxSimpleVariable(name, 0); break; // stem name. case STRING_STEM: return (RexxVariableBase *)new RexxStemVariable(name, 0); break; // compound name...more complicated. case STRING_COMPOUND_NAME: return (RexxVariableBase *)VariableDictionary::buildCompoundVariable(name, true); break; default: // this is invalid and a syntax error syntaxError(Error_Translation_invalid_attribute, name); } return OREF_NULL; } /** * Add a new instruction to the code execution chain. Instructions * are a linear chain of instruction objects, with all * branching handled by additional links within instructions. * * @param instruction * The new instruction to add. */ void LanguageParser::addClause(RexxInstruction *instruction) { // NOTE: the first instruction is set manually, so we should ALWAYS // have non-null values here. lastInstruction->setNext(instruction); lastInstruction = instruction; } /** * Add a label to our global table. Note, in Rexx it is * not an error to have duplicate labels, but only the * first can be used as a target. We do not overwrite * a given name if it is already in the table. * * @param label The label instruction to add. * @param labelname The label name. */ void LanguageParser::addLabel(RexxInstruction *label, RexxString *labelname ) { if (!labels->hasIndex(labelname)) { labels->put(label, labelname); } } /** * Locate a label for a name match. * * @param labelname The label name. * * @return The label instruction, or OREF_NULL if the label doesn't exist */ RexxInstruction *LanguageParser::findLabel(RexxString *labelname) { // it's possible we don't have a label table. if (labels != OREF_NULL) { return(RexxInstruction *)labels->get(labelname); } return OREF_NULL; } /** * We're about to process a WHEN expression on a GUARD * instruction, so we need to turn on variable capture * so we know what exposed variables are referenced in the * expression. */ void LanguageParser::setGuard() { // create the guard variable table if it is not already created... // it would be unusual for this to already exist. if (guardVariables == OREF_NULL) { guardVariables = new_identity_table(); } } /** * Complete guard variable collection and return the table of * variable names. * * @return An array of the guard variable names. */ ArrayClass *LanguageParser::getGuard() { // get the indices as an array of names. ArrayClass *guards = guardVariables->makeArray(); // turn off collection by tossing the table. guardVariables = OREF_NULL; return guards; } /** * Parse out a "constant" expression for a Rexx instruction. * a constant expression is a literal string, a constant * symbol, or an expression enclosed in parentheses. Most * new instructions use constant expressions, since they * are not terminated by instruction subkeywords, and thus * more immune to breakage when new options are added. * * @return An object that can be used to evaluate this option * expression. */ RexxInternalObject *LanguageParser::parseConstantExpression() { // everything keys off of the first token. RexxToken *token = nextReal(); // just a literal token? if (token->isLiteral()) /* literal string expression? */ { // process this as text and return. This is automatically // protected from GC. return addText(token); } // could be a constant symbol like a number or an environment variable // like .nil or .true. else if (token->isConstant()) { return addText(token); } // is this missing entirely? else if (token->isEndOfClause()) { // we don't raise an error here because we don't know that instruction // or context where this is needed. Just back up one token and return // a NULL value. previousToken(); return OREF_NULL; } // only other option here is an expression in parens. If this // is not there, raise an error (this is a generic error, so // we can do this here. else if (!token->isLeftParen()) { syntaxError(Error_Invalid_expression_general, token); } else { // parse our a subexpression, terminating on the end of clause or // a right paren (the right paren is actually the required terminator) RexxInternalObject *exp = parseFullSubExpression(TERM_RIGHT); // now verify that the terminator token was a right paren. If not, // issue an error message using the original opening token so we know // which one is an issue. if (!nextToken()->isRightParen()) { syntaxErrorAt(Error_Unmatched_parenthesis_paren, token); } // protect the expression from GC and return it. holdObject(exp); return exp; } return OREF_NULL; } /** * Evaluate a "parenthetical" expression for REXX instruction * values. A parenthetical expression is an expression enclosed * in parentheses. * * @param start The starting token of the expression. This will be * the opening paren and is used for error reporting. * * @return The parsed expression. */ RexxInternalObject *LanguageParser::parenExpression(RexxToken *start) { // NB, the opening paren has already been parsed off and is in // the start token, which we only need for error reporting. RexxInternalObject *expression = parseSubExpression(TERM_RIGHT); // this must be terminated by a right paren if (!nextToken()->isRightParen()) { syntaxErrorAt(Error_Unmatched_parenthesis_paren, start); } // protect the expression and return it. holdObject(expression); return expression; } /** * Parse off an expression, stopping when one of the possible set * of terminator tokens is reached. The terminator token is * placed back on the token queue. * * @param terminators * An int containing the bit flag versions of the terminators. * * @return An executable expression object. */ RexxInternalObject *LanguageParser::parseExpression(int terminators) { // get the first real token. This will skip over all of the // white space, comments, etc. to get to the real meat of the expression. // we then back up so that the real expression processing will see that as // the first token. nextReal(); previousToken(); // and go parse this. In this context, commas are allow to create // list format. return parseFullSubExpression(terminators); } /** * Perform the parsing of an expression where the expression * can be treated as a comma-separated list of subexpressions. * If we have just a simple single subexpression, the * return value is the parsed subexpression. If a comma * is found as a terminator, then we turn this expression * into an operator that will create an array object from the * list of expressions. Omitted expressions are allowed and * no effort is made to remove trailing null expressions. * * @param terminators * The list of terminators for this expression type. * * @return Either a simple expression, or an expression object for * creating an array item. */ RexxInternalObject *LanguageParser::parseFullSubExpression(int terminators) { size_t total = 0; // total is the full count of arguments we attempt to parse. RexxToken *terminatorToken; // the terminator token that ended a sub expression // now loop until we get a terminator. Note that COMMAs are always a terminator // token now that list expressions are possible. for (;;) { // parse off an argument expression RexxInternalObject *subExpr = parseSubExpression(terminators); // We have two term stacks. The main term stack is used for expression evaluation. // the subTerm stack is used for processing expression lists like this. // NOTE that we need to use pushSubTerm here so that the required expression stack // calculation comes out right. pushSubTerm(subExpr); // now check the total. Real count will be the last // expression that requires evaluation. total++; // the next token will be our terminator. If this is not // a comma, we have more expressions to parse. terminatorToken = nextToken(); if (!terminatorToken->isComma()) { // push this token back and stop parsing previousToken(); break; } } // if we only saw the single expression, then return that expression // as the result if (total == 1) { return popSubTerm(); } // we have an array creation list, so create the operator type for // building the array. return new (total) RexxExpressionList(total, subTerms); } /** * Parse off a sub- expression, stopping when one of the possible * set of terminator tokens is reached. The terminator token is * placed back on the token queue. This is generally * called in the context of parsing different bits of * an expression and is frequently called recursively. Note * that this subexpression parse will always terminate on a * comma...the calling context will then determine how that * comma is processed. * * @param terminators * an int containing flags that indicate what terminators * should be used to stop parsing this subexpression. * * @return */ RexxInternalObject *LanguageParser::parseSubExpression(int terminators ) { // generally, expressions proceed with term-operator-term, with various modifications. // start by processing of a term value. RexxInternalObject *left = parseMessageSubterm(terminators); RexxInternalObject *right = OREF_NULL; // hmmm, nothing found, so we're done. if (left == OREF_NULL) { return OREF_NULL; } // we keep both a stack of expression terms and one of operators and process // things based on precedence. // pus the term to the stack pushTerm(left); // add a fence value to the operator stack to wall us off from other // active subexpressions. pushOperator((RexxToken *)TheNilObject); // now see what is next. RexxToken *token = nextToken(); // loop until we find a terminator in our current token while (!token->isTerminator(terminators)) { switch (token->type()) { // either a single or double message send operation. // this uses a term from the stack as a target, and can possibly // have an argument list attached. case TOKEN_TILDE: case TOKEN_DTILDE: { // there must be a left term here to act as the target left = requiredTerm(token); // create a message term from the target...this also figures out // the message name and any arguments. RexxInternalObject *subexpression = parseMessage(left, token->isType(TOKEN_DTILDE), terminators); // that goes back on the term stack pushTerm(subexpression); break; } // a left square bracket. This will turn into a "[]" message. case TOKEN_SQLEFT: { // the term is required left = requiredTerm(token); // this is a message to the left term, and may (ok, probably) have // arguments as well RexxInternalObject *subexpression = parseCollectionMessage(token, left); // and this goes back on to the term stack pushTerm(subexpression); break; } // we have a symbol, which may be a variable, a literal string, or // an open paren, which may be the start of a parenthetical sub-expression // because we're looking for an operator at this point, this is actually // an implied abuttal operation. case TOKEN_SYMBOL: case TOKEN_LITERAL: case TOKEN_LEFT: { // get the token location. We're going to turn this into a zero-length // token for the abuttal operation. SourceLocation location = token->getLocation(); location.setEnd(location.getLineNumber(), location.getOffset()); // create a dummy token, push the abuttal token back on the queue, and fall // through to the next section where the blank concatenate operator is handled. token = new RexxToken (TOKEN_OPERATOR, location, OPERATOR_ABUTTAL, GlobalNames::NULLSTRING); previousToken(); } // NOTE: The above section falls through to this piece // A blank within an expression. This is a concatenate (although, if we've // fallen through from the section above, this is really an abuttal operation) case TOKEN_BLANK: { // this potentially could be a blank before one of the terminators (for example, // a blank before a WHILE keyword on a LOOP. This is the same with abuttal. // We need to check for the terminators, and if we get a hit, we're done here. RexxToken *second = nextReal(); // back up to the operator token and fall through to the operator logic. // this will then sort things out based on the sub type of the token. // we need to go to the previous token even if the terminator test is true. previousToken(); // not a real operator if adjacent to a terminator if (second->isTerminator(terminators)) { break; } } // we have an operator of some sort (including the blank and abuttal concatenates) // At this point, these will be dyadic operators, so we need to screen out the // the not operator, which can only be a prefix one. case TOKEN_OPERATOR: { // check for the backslash, which is a problem here. if (token->isSubtype(OPERATOR_BACKSLASH)) { syntaxError(Error_Invalid_expression_general, token); } // we now need to handle operator precedence with anything we have on the stack for (;;) { // get the top of the operator stack. If we hit the fence, we've // unwound any pending operations to the beginning, and we can quit this // loop. RexxToken *second = topOperator(); if (second == (RexxToken *)TheNilObject) { break; } // check the token precedence, If our new operator has higher // precedenc, we're done popping. if (token->precedence() > second->precedence()) { break; } // these are both required terms here. right = requiredTerm(token); left = requiredTerm(token); // pop off the top operator, and push a new term on the stack to // replace this. RexxToken *op = popOperator(); pushTerm(new RexxBinaryOperator(op->subtype(), left, right)); } // finished popping lower precedence items. Now push this operator on // to the stack. We need to evaluate its right-hand side before this can be handled. pushOperator(token); // evaluate a right-hand side to this expression. right = parseMessageSubterm(terminators); // Ok, we need a right term here, unless this is actually a blank operator. // in theory, we've already handled that possibility, but it doesn't hurt // to check. if (right == OREF_NULL && !token->isSubtype(OPERATOR_BLANK)) { syntaxError(Error_Invalid_expression_general, token); } // add the term to the stack. We can handle this once the // operator gets popped off of the stack. pushTerm(right); break; } // this one of the assignment operators, out of location. This is just // reported as an error here. case TOKEN_ASSIGNMENT: { syntaxError(Error_Invalid_expression_general, token); break; } // We've encountered a right paren. Like the comma, if this is // expected, it acts as a terminator. This is a stray and must // be reported. case TOKEN_RIGHT: { syntaxError(Error_Unexpected_comma_paren); break; } // another good character in a bad location. case TOKEN_SQRIGHT: { syntaxError(Error_Unexpected_comma_bracket); break; } // something unexpected that we have no specific error for. default: { syntaxError(Error_Invalid_expression_general, token); break; } } // ok, grab the next token and keep processing. token = nextToken(); } // step back so the terminator is the next token. previousToken(); // we've hit a terminator for this subexpression. We probably // have some pending operators on the operator stack that we need // to handle. These will be in proper precedence order, so we just // need to pop of the operators and terms and handle them. token = popOperator(); // keep popping until we hit our fence value while (token != (RexxToken *)TheNilObject) { // we need two terms right = requiredTerm(token); left = requiredTerm(token); // all of these operators are binaries, and get pushed back on the stack pushTerm(new RexxBinaryOperator(token->subtype(), left, right)); // and pop another operator. token = popOperator(); } // our final subexpression is at the top of the term stack, so pop it // off and return. return popTerm(); } /** * Parse off multiple arguments for a some sort of * call, function, or method invocation, returning them * as an array. * * @param firstToken The first token of the expresssion. * @param terminators * The set of terminators that will end the list. * * @return An array object holding the argument expressions. */ ArrayClass *LanguageParser::parseArgArray(RexxToken *firstToken, int terminators ) { // scan a set of arguments until we hit our terminator (likely to be // a ')' or ']'. Arguments are delimited by ',' size_t argCount = parseArgList(firstToken, terminators); // The arguments are pushed on to the term stack. We need to allocate // an array and copy them into the array ArrayClass *args = new_array(argCount); // arguments are pushed on to the term stack in reverse order, so // we fill from the end. Note that missing arguments are // just pushed on to the stack as OREF_NULL. while (argCount > 0) { args->put(subTerms->pop(), argCount--); } return args; } /** * Perform the parsing of a list of arguments where each * argument is separated by commas. Each argument expression * is pushed on to the term stack in last-to-first order, * with omitted arguments represented by OREF_NULL. * * @param firstToken The first token of the expression list, which is * really the delimiter that opens the list. * @param terminators * The list of terminators for this expression type. * * @return The count of argument expressions we found. Note that * an empty expression following a final "," does not * count as a real argument. */ size_t LanguageParser::parseArgList(RexxToken *firstToken, int terminators ) { size_t realcount = 0; // real count is arguments up to the last real one size_t total = 0; // total is the full count of arguments we attempt to parse. RexxToken *terminatorToken; // the terminator token that ended a sub expression // we need to skip ahead to the first real token, then backup one to be // properly positioned for the start. If this is a CALL instruction, this // will skip the blank between the CALL keyword and the start of the expression, // which counts as a significant blank by the tokenizer. nextReal(); previousToken(); // now loop until we get a terminator. COMMAs are always subexpression terminators, // so we don't need to add anything additional to our terminator lists. We just // interpret them differently in this context. for (;;) { // parse off an argument expression RexxInternalObject *subExpr = parseSubExpression(terminators); // We have two term stacks. The main term stack is used for expression evaluation. // the subTerm stack is used for processing expression lists like this. // NOTE that we need to use pushSubTerm here so that the required expression stack // calculation comes out right. pushSubTerm(subExpr); // now check the total. Real count will be the last // expression that requires evaluation. total++; // if we got something back, this is our new real total. if (subExpr != OREF_NULL) { realcount = total; } // the next token will be our terminator. If this is not // a comma, we have more expressions to parse. terminatorToken = nextToken(); if (!terminatorToken->isComma()) { break; } } // if this is a function or method invocation, we're expecting this list to be // ended by a right parent. firstToken gives the position of the missing // left paren. if (terminators & TERM_RIGHT && !terminatorToken->isRightParen()) { syntaxErrorAt(Error_Unmatched_parenthesis_paren, firstToken); } // same deal with square brackets, different error message. if (terminators&TERM_SQRIGHT && !terminatorToken->isRightBracket()) { syntaxErrorAt(Error_Unmatched_parenthesis_square, firstToken); } // all arguments are on the subterm stack now. If we had trailing // commas on the list, pop off any of the extras while (total > realcount) { subTerms->pop(); total--; } // return the count of real argument expressions. return realcount; } /** * Perform the parsing of a list of expressions for a SELECT * CASE WHEN expression. This is a list of one or more * expressions separated by commas. Omitted expressions are not * allowed. Each argument expression is pushed on to the term * stack in last-to-first order. * * @param firstToken The first token of the expression list, which is * really the delimiter that opens the list. * @param terminators * The list of terminators for this expression type. * * @return The count of argument expressions we found. */ size_t LanguageParser::parseCaseWhenList(int terminators ) { size_t total = 0; // total is the full count of arguments we attempt to parse. // we need to skip ahead to the first real token, then backup one to be // properly positioned for the start. If this is a CALL instruction, this // will skip the blank between the CALL keyword and the start of the expression, // which counts as a significant blank by the tokenizer. nextReal(); previousToken(); // now loop until we get a terminator. COMMAs are always subexpression terminators, // so we don't need to add anything additional to our terminator lists. We just // interpret them differently in this context. for (;;) { // parse off an argument expression RexxInternalObject *subExpr = parseSubExpression(terminators); // all sub expressions are required here. if (subExpr == OREF_NULL) { syntaxError(Error_Invalid_expression_case_when_list); } // We have two term stacks. The main term stack is used for expression evaluation. // the subTerm stack is used for processing expression lists like this. // NOTE that we need to use pushSubTerm here so that the required expression stack // calculation comes out right. pushSubTerm(subExpr); // now check the total. total++; // the next token will be our terminator. If this is not // a comma, we have more expressions to parse. RexxToken *terminatorToken = nextToken(); if (!terminatorToken->isComma()) { // put the terminator back on previousToken(); break; } } // return the count of expressions. return total; } /** * Parse off a function call and create an expression * object that can process the call. * * @param token The token that marks the start of the argument list. * @param name The function name token. * * @return A function expression object that can invoke this function. */ RexxInternalObject *LanguageParser::parseFunction(RexxToken *token, RexxToken *name) { // parse off the argument list, leaving the arguments in the subterm stack. // NOTE: Because we have a closed () construct delimiting the function arguments, // we can ignore any terminators specified from the parent context. size_t argCount = parseArgList(token, (TERM_RIGHT)); // create a function item. This will also pull the argument items from the // subterm stack RexxInternalObject *func = new (argCount) RexxExpressionFunction(name->value(), argCount, subTerms, name->builtin()); // at this point, we can't resolve the final target of this call. It could be // a builtin, a call to an internal label, or an external call. We'll resolve this // once we've finished this code block and have all of the labels. But note that // if this was specified as a literal, we can skip the resolution step since // this can never resolve to an internal label. if (!name->isLiteral()) { addReference(func); } return func; } /** * Parse off a qualified symbol. This can either be a class * lookup or a qualified function call. * the call. * * @param n * * @return An expression object that can process this qualified lookup * type. */ RexxInternalObject *LanguageParser::parseQualifiedSymbol(RexxString *namespaceName) { RexxToken *token = nextToken(); if (!token->isSymbol()) { syntaxError(Error_Symbol_expected_qualified_symbol); } // get the qualified name RexxString *qualifiedName = token->value(); // step to the next immediate token. If this is left paren, then // this is a qualified function call token = nextToken(); if (token->isLeftParen()) { // parse off the argument list, leaving the arguments in the subterm stack. // NOTE: Because we have a closed () construct delimiting the function arguments, // we can ignore any terminators specified from the parent context. size_t argCount = parseArgList(token, (TERM_RIGHT)); // create a function item. This will also pull the argument items from the // subterm stack return new (argCount) QualifiedFunction(namespaceName, qualifiedName, argCount, subTerms); } // this is a qualified class lookup else { // need to push the following token back...it is something else. previousToken(); return new ClassResolver(namespaceName, qualifiedName); } } /** * Parse a collection message expression term. This is * if the form term[args], and follows much the same * parsing rules as function calls. * * @param token The token that starts the argument list (the square bracket) * @param target The target for the message term. * @param terminators * Our terminators. * * @return A message expression object. */ RexxInternalObject *LanguageParser::parseCollectionMessage(RexxToken *token, RexxInternalObject *target) { // this was popped from the term stack, so we need to give it a little protection // until we're done parsing. ProtectedObject p(target); // get the arguments. Like with builtin function calls, we just ignore any // prior terminator context and rely on the fact that the brackets must match. size_t argCount = parseArgList(token, (TERM_SQRIGHT)); // create the message item. RexxInternalObject *msg = new (argCount) RexxExpressionMessage(target, GlobalNames::BRACKETS, OREF_NULL, argCount, subTerms, false); // give this a little short-term GC protection. holdObject(msg); return msg; } /** * Get a token from the clause, combining terminator checks * and error reporting if we don't get a terminator. * * @param terminators * The terminators for the current context. * @param errorcode The error to issue if no token is found. The error code * is optional, and if not specified, OREF_NULL is returned. * * @return The parsed off token. */ RexxToken *LanguageParser::getToken(int terminators, RexxErrorCodes errorcode) { // get the next token and perform the terminator checks. If we were // terminated, issue an error message if requested. RexxToken *token = nextToken(); if (token->isTerminator(terminators)) { if (errorcode != 0) { syntaxError(errorcode); } // just return a null if not requested to issue an error. previousToken(); return OREF_NULL; } return token; } /** * Parse a message send term within an expression. * * @param target The target object term for the message send. * @param doubleTilde * Indicates whether this is the "~" or "~~" form of operation. * @param terminators * Expression terminators. * * @return An object to execute the message send. */ RexxInternalObject *LanguageParser::parseMessage(RexxInternalObject *target, bool doubleTilde, int terminators ) { // this message might have a superclass override...default is none. RexxInternalObject *super = OREF_NULL; // no arguments yet size_t argCount = 0; // add the term to the term stack so that stacksize calculations // include this in the processing. This has the side effect of // protecting this object from GC while we're parsing. pushTerm(target); // ok, we're expecting a message name next, go get one. RexxToken *token = getToken(terminators, Error_Symbol_or_string_tilde); // this must be a string or symbol if (!token->isSymbolOrLiteral()) { syntaxError(Error_Symbol_or_string_tilde); } // we have a message name RexxString *messagename = token->value(); // upper case this here and add to the common pool. messagename = commonString(messagename->upper()); // ok, what else do we have. Nothing additional is required // here, but this could be an argument list, or a superclass // override. token = getToken(terminators); if (token != OREF_NULL) { // an override is marked with a colon. if (token->isType(TOKEN_COLON)) { // ok, if we have the colon, then we need an override spec. token = getToken(terminators, Error_Symbol_expected_colon); // this must be a variable or a dot symbol. if (!token->isVariableOrDot()) { syntaxError(Error_Symbol_expected_colon); } // the super class is either a variable lookup or a dot variable, so // we need a retriever. super = addText(token); // get the next token to check for an argument list. token = getToken(terminators); // NOTE: We fall through from here to the code // that follows the checks for the superclass override to // the code that checks for the argument list. } } // ok, we might have an argument list here. if (token != OREF_NULL) { // this is only interesting if the text token is an open paren directly // abutted to the name. Process that as an argument list for the final // created message object if (token->isLeftParen()) { // NOTE: because of the parens, we can ignore our parent terminators... // we only look for the right paren argCount = parseArgList(token, TERM_RIGHT); } else { previousToken(); /* something else, step back */ } } // got all of the pieces, now create the message object and give it some short term // protection from GC. RexxInternalObject *msg = new (argCount) RexxExpressionMessage(target, messagename, super, argCount, subTerms, doubleTilde); holdObject(msg); // now safe to pop the message target object we saved. popTerm(); return msg; } /** * Parse off a single variable symbol or a message term that * can be used for an assignment. * * NOTE: If this is a message term, then the message term * will be configured as an assignment term. * * @return The object for an assignment target, or OREF_NULL if something * other than a variable or a message term was found. On return, * the clause position pointer will either be unchanged or * positioned at the next token of the clause. */ RexxInternalObject *LanguageParser::parseVariableOrMessageTerm() { // try for a message term first. If not successful, see if the // next token is a variable symbol. RexxInternalObject *result = parseMessageTerm(); if (result == OREF_NULL) { RexxToken *_first = nextReal(); if (_first->isSymbol()) { // ok, add the variable to the processing list needVariable(_first); result = addText(_first); } else { previousToken(); // just push back on for the caller to sort out } } else { // we need to convert this into an assignment message. ((RexxExpressionMessage *)result)->makeAssignment(this); } return result; } /** * Parse off an instruction leading message term element. This * is an important step, as it is used to determine what sort of * instruction we have. If the instruction starts with a * message term, then this is a messaging instruction. Part of * the instruction type determination at the beginning of each * clause. * * @return The message term execution element. */ RexxInternalObject *LanguageParser::parseMessageTerm() { // save the current position so we can reset cleanly size_t mark = markPosition(); // The straight forward approach is to parse off the first // sub term and then see if this is followed by a messaging // operator. Unfortunately, there's a problem hidden within // that approach. Parsing off the first term results in an // symbols that are located getting handled as variable references. // This means that for all keyword instructions, we're allocating a variable // slot and creating a variable retriever for all keyword instruction names // used within a code block. And this happens with every code block (e.g., // ::method or ::routine section) in the program. So, we will attempt to // eliminate this particular situation up front just be examining the // tokens. // get the first token and make sure we really have something here. // The caller knows how to deal with a missing term. RexxToken *token = nextToken(); if (token->isTerminator(TERM_EOC)) { // push the terminator back previousToken(); return OREF_NULL; } // this problem only occurs if the first token is a simple variable token. if (token->isSimpleVariable()) { RexxToken *second = nextToken(); // we can go ahead and reset...we need to do this regardless of the // path we take. resetPosition(mark); // if the first token is a symbol that is followed by a // message operator token (~, ~~, or [), this is a message term. // if the next token is a "(", this is "potentially" a function call // that is a message term. But in that case, it will not be handled // as a variable. For any other case, we reject this as a message term // and return NULL. if (!second->isMessageOperator() & !second->isLeftParen()) { return OREF_NULL; } // fall through and do the generalized parsing process from here. } // if not a simple symbol, back up to the first token again else { resetPosition(mark); } // get the first message term RexxInternalObject *start = parseSubTerm(TERM_EOC); // save this on the term stack pushTerm(start); RexxInternalObject *term = OREF_NULL; // an allocated message term token = nextToken(); // the leading message term can be a cascade of messages, so // keep processing things until we hit some other type of operation. while (token->isMessageOperator()) { // we need to perform the the message operation on this term to create // an new term which will be the target of the next one. // this could be a bracket lookup, which is a collection message. if (token->isLeftBracket()) { term = parseCollectionMessage(token, start); } else { // normal message twiddle message term = parseMessage(start, token->isType(TOKEN_DTILDE), TERM_EOC); } // this is the target for the next one in the chain. // remove the previous one from the term stack and push a new // one on. start = term; popTerm(); pushTerm(start); // and see if we have another. token = nextToken(); } // found a token that was not a message send type, so push it back. previousToken(); // if we never created a message term from this, we need to take another // path. Roll the position back to where we started and return. if (term == OREF_NULL) { resetPosition(mark); } // we still have either the starting term or our last created message // term on the stack...we can remove this now. popTerm(); // ok, this is either OREF_NULL to indicate we found nothing, // or the last message term we ended up creating. return term; } /** * Handle a potential message subterm in the context of an * expression. This is similar to messageTerm(), but this * occurs after we've already determined we need an expression * and need an expression term. * * @param terminators * Terminators for this subexpression context. * * @return An object to represent this message term. Returns * OREF_NULL if no suitable term is found. */ RexxInternalObject* LanguageParser::parseMessageSubterm(int terminators) { // with very complex instructions, it is possible to recurse quite deeply here. // so give a look at the stack space on each call so we can terminate "nicely" ActivityManager::currentActivity->checkStackSpace(); // get the first token. If we've hit a terminator here, this could be // the real end of the expression. The caller context will figure out // how to handle that. RexxToken *token = nextToken(); if (token->isTerminator(terminators)) { // need to push the terminator back previousToken(); return OREF_NULL; } // is this an operator? I this context, it will need to be a prefix operator. if (token->isOperator()) { // prefix operators are handled as expression terms, although they themselves // require a term. switch (token->subtype()) { // we only recognize 3 prefix operators, and they are handled in pretty // much the same way. case OPERATOR_PLUS: case OPERATOR_SUBTRACT: case OPERATOR_BACKSLASH: { // we need a term as a target. this is an error // if we don't find anything we can use. We just recurse // on this, which might find a chain of prefix operators (except, // as I'm sure Walter Pachl will be sure to point out, --, which // will be treated as a line comment :-) RexxInternalObject *term = parseMessageSubterm(terminators); if (term == OREF_NULL) { syntaxError(Error_Invalid_expression_prefix, token); } // create a new unary operator using the subtype code. return new RexxUnaryOperator(token->subtype(), term); break; } // not an operator in the normal sense, but > or as a prefix creates // a variable reference. case OPERATOR_LESSTHAN: case OPERATOR_GREATERTHAN: { // parse off the term return parseVariableReferenceTerm(); } // other operators are invalid default: syntaxError(Error_Invalid_expression_general, token); } } // ok, not an operator character. Now we need to figure out what sort of // term element is here. else { // put the token back and try parsing off a subTerm (smaller subset, no // prefix stuff allowed). previousToken(); RexxInternalObject *term = parseSubTerm(terminators); // protect on the term stack pushTerm(term); // ok, now see if this is actually a message send target by checking // the next token token = nextToken(); // we can have a long cascade of message sends. For expression syntax, // this all one term (just like nested function calls would be). while (token->isMessageOperator()) { // we have two possibilities here, a bracket message or a twiddle form. if (token->isLeftBracket()) { term = parseCollectionMessage(token, term); } else { term = parseMessage(term, token->isType(TOKEN_DTILDE), TERM_EOC); } popTerm(); pushTerm(term); // message cascades are considered part of the same expression term. token = nextToken(); } // back up to the token that stopped the loop previousToken(); // pop our term from the stack and return the final version. popTerm(); return term; } return OREF_NULL; // should never get here. } /** * Parse off a variable reference term. * * @return A variable reference term object */ RexxInternalObject* LanguageParser::parseVariableReferenceTerm() { // this must be either a simple variable or a stem. RexxToken *token = nextReal(); if (!token->isSymbol() || !token->isNonCompoundVariable()) { syntaxError(Error_Symbol_expected_after_prefix_reference, token); } RexxVariableBase *retriever = OREF_NULL; if (token->isSimpleVariable()) { retriever = addSimpleVariable(token->value()); } else { retriever = addStem(token->value()); } // create a new expression term to retrieve the variable return new VariableReferenceOp(retriever); } /** * Parse off a subterm of an expression, from simple ones like * variable names and literals, to more complex such as message * sends. * * @param terminators * The expression terminator context. * * @return An executable object for the message term. */ RexxInternalObject* LanguageParser::parseSubTerm(int terminators) { // with very complex instructions, it is possible to recurse quite deeply here. // so give a look at the stack space on each call so we can terminate "nicely" ActivityManager::currentActivity->checkStackSpace(); // get the first token and make sure we really have something here. // The caller knows how to deal with a missing term. RexxToken *token = nextToken(); if (token->isTerminator(terminators)) { // push the terminator back previousToken(); return OREF_NULL; } // ok, process based on the token category switch (token->type()) { // parenthetical subexpression. Note, we do not pass along our // terminators when requesting this, since the bracketing within the parens // will disambiguate keywords. We only look for terminators and a right paren case TOKEN_LEFT: { // parse off the parenthetical. This might not return anything if there // is nothing in the parens. This is an error. Also, in this context, // we are back in a mode where the array-creation syntax is allowed. RexxInternalObject *term = parseFullSubExpression(TERM_RIGHT); if (term == OREF_NULL) { syntaxError(Error_Invalid_expression_general, token); } // this had better been terminated by a right paren. if (!nextToken()->isRightParen()) { syntaxErrorAt(Error_Unmatched_parenthesis_paren, token); } // we're done. return term; } // a symbol. These are generally pretty simple, but // we also have to account for function calls or qualified lookups case TOKEN_SYMBOL: { // need to check if the next token is an open paren. That turns // the symbol or literal token into a function invocation. RexxToken *second = nextToken(); if (second->isLeftParen()) { return parseFunction(second, token); } // either a qualified symbol lookup or a potential // qualified function. else if (second->isType(TOKEN_COLON)) { return parseQualifiedSymbol(token->value()); } else { // simple text token type...push the next token back // and go resolve how the token is handled. previousToken(); return addText(token); } break; } // a literal. These are generally pretty simple, but // we also have to account for function calls. case TOKEN_LITERAL: { // need to check if the next token is an open paren. That turns // the symbol or literal token into a function invocation. RexxToken *second = nextToken(); if (second->isLeftParen()) { return parseFunction(second, token); } else { // simple text token type...push the next token back // and go resolve how the token is handled. previousToken(); return addText(token); } break; } // an operator token. We do allow prefix operators here, others are // an error case TOKEN_OPERATOR: { switch (token->subtype()) { // +, -, and logical NOT variants are permitted here...except // we don't actually process them here, so back up and say we got nothing. case OPERATOR_PLUS: case OPERATOR_SUBTRACT: case OPERATOR_BACKSLASH: previousToken(); return OREF_NULL; // a prefix '>' or '<' is a variable reference case OPERATOR_LESSTHAN: case OPERATOR_GREATERTHAN: { // parse off the term return parseVariableReferenceTerm(); } // other operators we can flag as an error now. default: syntaxError(Error_Invalid_expression_general, token); } break; } // a few error situations with specific error messages. case TOKEN_RIGHT: syntaxError(Error_Unexpected_comma_paren); break; case TOKEN_COMMA: syntaxError(Error_Unexpected_comma_comma); break; case TOKEN_SQRIGHT: syntaxError(Error_Unexpected_comma_bracket); break; // ok, this is bad default: syntaxError(Error_Invalid_expression_general, token); } // never reach here. return OREF_NULL; } /** * Push a term on to the expression term stack. * * @param term The term object. */ void LanguageParser::pushTerm(RexxInternalObject *term ) { // push the term on to the stack. terms->push(term); // we keep track of how large the term stack gets during parsing. This // tells us how much stack space we need to allocate at run time. currentStack++; maxStack = std::max(currentStack, maxStack); } /** * Pop a term off of the expression stack. * * @return The popped object. */ RexxInternalObject *LanguageParser::popTerm() { // reduce the stack count currentStack--; // pop the object off of the stack and give it some short-term // GC protection. RexxInternalObject *term = terms->pop(); holdObject(term); return term; } /** * Push a term on to the expression sub term stack. The * subterms normally contribute to the total required stack * size, so make sure we account for these when calculating the * total required stack size. Only use this method of pushing * the term when the max stack size is affected. * * @param term The term object. */ void LanguageParser::pushSubTerm(RexxInternalObject *term ) { // push the term on to the stack. subTerms->push(term); // we keep track of how large the term stack gets during parsing. This // tells us how much stack space we need to allocate at run time. currentStack++; maxStack = std::max(currentStack, maxStack); } /** * Pop a term off of the expression sub term stack. * * @return The popped object. */ RexxInternalObject *LanguageParser::popSubTerm() { // reduce the stack count currentStack--; // pop the object off of the stack and give it some short-term // GC protection. RexxInternalObject *term = subTerms->pop(); holdObject(term); return term; } /** * Pop a term item off of the execution stack. Will * raise an error if this is not there. * * @param token The token giving the context location. Used to * identify the error position. * @param errorCode The error code. The default is the generic syntax error. * * @return The object popped off of the stack. */ RexxInternalObject *LanguageParser::requiredTerm(RexxToken *token, RexxErrorCodes errorCode) { // we track the size count when we push/pop currentStack--; // pop the term off of the stack RexxInternalObject *term = terms->pop(); // we need a term, if this is not here, this is a syntax error if (term == OREF_NULL) { syntaxError(errorCode, token); } // give this term some short-term GC protection and return it. holdObject(term); return term; } /** * Pop multiple items off of the term stack. * * @param count The count of items to pop. * * @return The last object popped. */ RexxInternalObject *LanguageParser::popNTerms(size_t count) { RexxInternalObject *result = OREF_NULL; // reduce the stack size currentStack -= count; // and pop that many elements while (count--) { result = terms->pop(); } // if we have a return item, protect it for a little while. if (result != OREF_NULL) { holdObject(result); } return result; /* and return it */ } /** * Break up a string into an array of words for parsing and * interpretation. The first word will be uppercased, and all * of the strings will be added to the common string pool. * * @param string The string we break up. * * @return An array of the words. */ ArrayClass *LanguageParser::words(RexxString *string) { // reduce this to an array of words... this is the easy part. ArrayClass *wordArray = string->subWords(OREF_NULL, OREF_NULL); size_t count = wordArray->items(); // it's possible this could contain nothing...just return the // empty array if (count == 0) { return wordArray; } // we could be creating new objects here, better protect this ProtectedObject p(wordArray); // we know we have at least 1 word here. Replace the first one // with the uppercase, commonstring version. wordArray->put(commonString(((RexxString *)wordArray->get(1))->upper()), 1); // now make commonstring versions of the rest of the words for (size_t i = 2; i <= count; i++) { wordArray->put(commonString(((RexxString *)wordArray->get(i))), i); } return wordArray; } /** * Raise a simple error message, using the current clause * location information. * * @param errorcode The error message to raise. */ void LanguageParser::error(RexxErrorCodes errorcode) { ActivityManager::currentActivity->raiseException(errorcode, OREF_NULL, OREF_NULL, OREF_NULL); } /** * Raise an error message using specifically provided location * information. * * @param errorcode The error message to raise. * @param location The location of the instruction in error. * @param subs The message substitutions. */ void LanguageParser::error(RexxErrorCodes errorcode, const SourceLocation &location, ArrayClass *subs) { // set the error location. This location is picked up from the // parse context stack frame we set up before we started clauseLocation = location; ActivityManager::currentActivity->raiseException(errorcode, OREF_NULL, subs, OREF_NULL); } /** * Raise an error where one of the error message substitutions * is the line number location of another instruction. * * @param errorcode The error to issue * @param instruction * The instruction used to obtain the line number for the * message. */ void LanguageParser::errorLine(RexxErrorCodes errorcode, RexxInstruction *instruction) { ActivityManager::currentActivity->raiseException(errorcode, OREF_NULL, new_array(new_integer(instruction->getLineNumber())), OREF_NULL); } /** * Raise an error using a Token location for the message * substitution. * * @param errorcode The error to issue. * @param token The source token for the location. */ void LanguageParser::errorPosition(RexxErrorCodes errorcode, RexxToken *token ) { SourceLocation tokenLocation = token->getLocation(); ActivityManager::currentActivity->raiseException(errorcode, OREF_NULL, new_array(new_integer(tokenLocation.getOffset() + 1), new_integer(tokenLocation.getLineNumber())), OREF_NULL); } /** * Raise an error using the value of the token as a * substitution parameter. * * @param errorcode The error code to issue. * @param token The token used for the substitution information. */ void LanguageParser::errorToken(RexxErrorCodes errorcode, RexxToken *token ) { ActivityManager::currentActivity->raiseException(errorcode, OREF_NULL, new_array(token->displayValue()), OREF_NULL); } /** * Issue an error message with a single substitution value. * * @param errorcode The error to issue. * @param value The substitution value. */ void LanguageParser::error(RexxErrorCodes errorcode, RexxObject *value ) { ActivityManager::currentActivity->raiseException(errorcode, OREF_NULL, new_array(value), OREF_NULL); } /** * Issue an error message with two substitution values. * * @param errorcode The error number to issue. * @param value1 The first substitution object. * @param value2 The second substitution object. */ void LanguageParser::error(RexxErrorCodes errorcode, RexxObject *value1, RexxObject *value2 ) { ActivityManager::currentActivity->raiseException(errorcode, OREF_NULL, new_array(value1, value2), OREF_NULL); } /** * Issue an error message with three substitution values. * * @param errorcode The error number to issue. * @param value1 The first substitution object. * @param value2 The second substitution object. * @param value3 The third substitution object. */ void LanguageParser::error(RexxErrorCodes errorcode, RexxObject *value1, RexxObject *value2, RexxObject *value3 ) { ActivityManager::currentActivity->raiseException(errorcode, OREF_NULL, new_array(value1, value2, value3), OREF_NULL); } /** * Raise an error for an unclosed block instruction. * * @param instruction * The unclosed instruction. */ void LanguageParser::blockError(RexxInstruction *instruction) { // get the last instruction location and set as the current error location clauseLocation = lastInstruction->getLocation(); switch (instruction->getType()) { // each type of block instruction has its own message // DO instruction case KEYWORD_SIMPLE_BLOCK: case KEYWORD_DO: syntaxError(Error_Incomplete_do_do, instruction); break; // LOOP instruction case KEYWORD_LOOP: case KEYWORD_LOOP_FOREVER: case KEYWORD_LOOP_OVER: case KEYWORD_LOOP_OVER_UNTIL: case KEYWORD_LOOP_OVER_WHILE: case KEYWORD_LOOP_OVER_FOR: case KEYWORD_LOOP_OVER_FOR_UNTIL: case KEYWORD_LOOP_OVER_FOR_WHILE: case KEYWORD_LOOP_CONTROLLED: case KEYWORD_LOOP_CONTROLLED_UNTIL: case KEYWORD_LOOP_CONTROLLED_WHILE: case KEYWORD_LOOP_COUNT: case KEYWORD_LOOP_COUNT_UNTIL: case KEYWORD_LOOP_COUNT_WHILE: case KEYWORD_LOOP_WHILE: case KEYWORD_LOOP_UNTIL: case KEYWORD_LOOP_WITH: case KEYWORD_LOOP_WITH_UNTIL: case KEYWORD_LOOP_WITH_WHILE: case KEYWORD_LOOP_WITH_FOR: case KEYWORD_LOOP_WITH_FOR_UNTIL: case KEYWORD_LOOP_WITH_FOR_WHILE: syntaxError(Error_Incomplete_do_loop, instruction); break; // SELECT instruction case KEYWORD_SELECT: case KEYWORD_SELECT_CASE: syntaxError(Error_Incomplete_do_select, instruction); break; // OTHERWISE section of a SELECT case KEYWORD_OTHERWISE: syntaxError(Error_Incomplete_do_otherwise, instruction); break; // different variants of an IF case KEYWORD_IF: case KEYWORD_WHEN: case KEYWORD_WHEN_CASE: case KEYWORD_IFTHEN: case KEYWORD_WHENTHEN: syntaxError(Error_Incomplete_do_then, instruction); break; // ELSE problem case KEYWORD_ELSE: syntaxError(Error_Incomplete_do_else, instruction); break; // invalid block instruction type (should really never happen) default: { reportException(Error_Interpretation_switch, "block instruction type", instruction->getType()); break; } } } /** * Parse off a "logical list expression", consisting of a * list of conditionals separated by commas. * * @return OREF_NULL if no expressions is found, a single expression * element if a single expression is located, and a complex * logical expression operator for a list of expressions. */ RexxInternalObject *LanguageParser::parseLogical(int terminators) { // These are not delimited lists, but are part of keyword contexts where // other keywords terminate the expression (e.g., IF, WHEN, WHILE), so we // need to pass along the terminators. Unlike argument lists, // omitted expressions are not allowed: size_t total = 0; // total is the full count of arguments we attempt to parse. RexxToken *terminatorToken; // the terminator token that ended a sub expression // we need to skip ahead to the first real token, then backup one to be // properly positioned for the start. If this is an IF instruction, this // will skip the blank between the IF keyword and the start of the expression, // which counts as a significant blank by the tokenizer. nextReal(); previousToken(); // now loop until we get a terminator. Note that commas are always subexpression // terminators, but in this context, we interpret them differently. for (;;) { // parse off an argument expression RexxInternalObject *subExpr = parseSubExpression(terminators); // all sub expressions are required here. if (subExpr == OREF_NULL) { syntaxError(Error_Invalid_expression_logical_list); } // We have two term stacks. The main term stack is used for expression evaluation. // the subTerm stack is used for processing expression lists like this. pushSubTerm(subExpr); // add this to our total count. total++; // the next token will be our terminator. If this is not // a comma, we have more expressions to parse. terminatorToken = nextToken(); if (!terminatorToken->isComma()) { // push the terminator token back previousToken(); break; } } // we have at least one item (caught in the loop) and if we have exactly // one (most common situation), just pop the top item and return it if (total == 1) { return subTerms->pop(); } // composite tis expression into a single object that can evaluate the // multiple expressions. return new (total) RexxExpressionLogical(total, subTerms); } /** * Process handling of instore execution arguments. * * @param instore The instore descriptor. * @param name The name of the program. * * @return An executable top-level program object (a routine, actually) */ RoutineClass *LanguageParser::processInstore(PRXSTRING instore, RexxString * name ) { // just a generic empty one indicating that we should check the macrospace? if (instore[0].strptr == NULL && instore[1].strptr == NULL) { unsigned short temp; // see if this exists if (!RexxQueryMacro(name->getStringData(), &temp)) { return restoreFromMacroSpace(name); } return OREF_NULL; } // we have a precompiled image (likely a repeat call...restore and reuse this if (instore[1].strptr != NULL) { // we're saved as a program object Protected routine = RoutineClass::restore(&instore[1], name); if (!routine.isNull()) { // did it unflatten successfully? If we have source also, // reattach it to the routine for tracing/error reporting. if (instore[0].strptr != NULL) { Protected source_buffer = new_buffer(instore[0]); routine->getPackageObject()->attachSource(source_buffer); } return routine; /* go return it */ } } // we have instorage source, but no compiled image. We'll compile, and optionally // flatten the program for reuse. if (instore[0].strptr != NULL) { Protected source_buffer = new_buffer(instore[0]); // translate the source Protected routine = createProgram(name, source_buffer); // save this in the instore buffer routine->save(&instore[1]); return routine; } return OREF_NULL; /* processing failed */ } /** * Create a routine from a macrospace source. * * @param name The name of the macrospace item. * * @return The inflatted macrospace routine. */ RoutineClass *LanguageParser::restoreFromMacroSpace(RexxString *name) { // the instorage buffer RXSTRING buffer; MAKERXSTRING(buffer, NULL, 0); // get the image of function RexxResolveMacroFunction(name->getStringData(), &buffer); // unflatten the method now Protected routine = RoutineClass::restore(&buffer, name); // release the buffer memory SystemInterpreter::releaseResultMemory(buffer.strptr); return routine; }