610 lines
17 KiB
C
610 lines
17 KiB
C
/**
|
|
* Contains the default implementation of the common token used within
|
|
* java. Custom tokens should create this structure and then append to it using the
|
|
* custom pointer to install their own structure and API.
|
|
*/
|
|
|
|
// [The "BSD licence"]
|
|
// Copyright (c) 2005-2009 Jim Idle, Temporal Wave LLC
|
|
// http://www.temporal-wave.com
|
|
// http://www.linkedin.com/in/jimidle
|
|
//
|
|
// All rights reserved.
|
|
//
|
|
// Redistribution and use in source and binary forms, with or without
|
|
// modification, are permitted provided that the following conditions
|
|
// are met:
|
|
// 1. Redistributions of source code must retain the above copyright
|
|
// notice, this list of conditions and the following disclaimer.
|
|
// 2. Redistributions in binary form must reproduce the above copyright
|
|
// notice, this list of conditions and the following disclaimer in the
|
|
// documentation and/or other materials provided with the distribution.
|
|
// 3. The name of the author may not be used to endorse or promote products
|
|
// derived from this software without specific prior written permission.
|
|
//
|
|
// THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
|
|
// IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
|
|
// OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
|
|
// IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
|
|
// INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
|
|
// NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
|
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
|
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
|
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
|
|
// THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|
|
|
#include <antlr3.h>
|
|
|
|
/* Token API
|
|
*/
|
|
static pANTLR3_STRING getText (pANTLR3_COMMON_TOKEN token);
|
|
static void setText (pANTLR3_COMMON_TOKEN token, pANTLR3_STRING text);
|
|
static void setText8 (pANTLR3_COMMON_TOKEN token, pANTLR3_UINT8 text);
|
|
static ANTLR3_UINT32 getType (pANTLR3_COMMON_TOKEN token);
|
|
static void setType (pANTLR3_COMMON_TOKEN token, ANTLR3_UINT32 type);
|
|
static ANTLR3_UINT32 getLine (pANTLR3_COMMON_TOKEN token);
|
|
static void setLine (pANTLR3_COMMON_TOKEN token, ANTLR3_UINT32 line);
|
|
static ANTLR3_INT32 getCharPositionInLine (pANTLR3_COMMON_TOKEN token);
|
|
static void setCharPositionInLine (pANTLR3_COMMON_TOKEN token, ANTLR3_INT32 pos);
|
|
static ANTLR3_UINT32 getChannel (pANTLR3_COMMON_TOKEN token);
|
|
static void setChannel (pANTLR3_COMMON_TOKEN token, ANTLR3_UINT32 channel);
|
|
static ANTLR3_MARKER getTokenIndex (pANTLR3_COMMON_TOKEN token);
|
|
static void setTokenIndex (pANTLR3_COMMON_TOKEN token, ANTLR3_MARKER);
|
|
static ANTLR3_MARKER getStartIndex (pANTLR3_COMMON_TOKEN token);
|
|
static void setStartIndex (pANTLR3_COMMON_TOKEN token, ANTLR3_MARKER index);
|
|
static ANTLR3_MARKER getStopIndex (pANTLR3_COMMON_TOKEN token);
|
|
static void setStopIndex (pANTLR3_COMMON_TOKEN token, ANTLR3_MARKER index);
|
|
static pANTLR3_STRING toString (pANTLR3_COMMON_TOKEN token);
|
|
|
|
/* Factory API
|
|
*/
|
|
static void factoryClose (pANTLR3_TOKEN_FACTORY factory);
|
|
static pANTLR3_COMMON_TOKEN newToken (void);
|
|
static void setInputStream (pANTLR3_TOKEN_FACTORY factory, pANTLR3_INPUT_STREAM input);
|
|
static void factoryReset (pANTLR3_TOKEN_FACTORY factory);
|
|
|
|
/* Internal management functions
|
|
*/
|
|
static ANTLR3_BOOLEAN newPool (pANTLR3_TOKEN_FACTORY factory);
|
|
static pANTLR3_COMMON_TOKEN newPoolToken (pANTLR3_TOKEN_FACTORY factory);
|
|
|
|
|
|
ANTLR3_API pANTLR3_COMMON_TOKEN
|
|
antlr3CommonTokenNew(ANTLR3_UINT32 ttype)
|
|
{
|
|
pANTLR3_COMMON_TOKEN token;
|
|
|
|
// Create a raw token with the interface installed
|
|
//
|
|
token = newToken();
|
|
|
|
if (token != NULL)
|
|
{
|
|
token->setType(token, ttype);
|
|
}
|
|
|
|
// All good
|
|
//
|
|
return token;
|
|
}
|
|
|
|
ANTLR3_API pANTLR3_TOKEN_FACTORY
|
|
antlr3TokenFactoryNew(pANTLR3_INPUT_STREAM input)
|
|
{
|
|
pANTLR3_TOKEN_FACTORY factory;
|
|
|
|
/* allocate memory
|
|
*/
|
|
factory = (pANTLR3_TOKEN_FACTORY) ANTLR3_MALLOC((size_t)sizeof(ANTLR3_TOKEN_FACTORY));
|
|
|
|
if (factory == NULL)
|
|
{
|
|
return NULL;
|
|
}
|
|
|
|
/* Install factory API
|
|
*/
|
|
factory->newToken = newPoolToken;
|
|
factory->close = factoryClose;
|
|
factory->setInputStream = setInputStream;
|
|
factory->reset = factoryReset;
|
|
|
|
/* Allocate the initial pool
|
|
*/
|
|
factory->thisPool = -1;
|
|
factory->pools = NULL;
|
|
factory->maxPool = -1;
|
|
newPool(factory);
|
|
|
|
/* Factory space is good, we now want to initialize our cheating token
|
|
* which one it is initialized is the model for all tokens we manufacture
|
|
*/
|
|
antlr3SetTokenAPI(&factory->unTruc);
|
|
|
|
/* Set some initial variables for future copying
|
|
*/
|
|
factory->unTruc.factoryMade = ANTLR3_TRUE;
|
|
|
|
// Input stream
|
|
//
|
|
setInputStream(factory, input);
|
|
|
|
return factory;
|
|
|
|
}
|
|
|
|
static void
|
|
setInputStream (pANTLR3_TOKEN_FACTORY factory, pANTLR3_INPUT_STREAM input)
|
|
{
|
|
factory->input = input;
|
|
factory->unTruc.input = input;
|
|
if (input != NULL)
|
|
{
|
|
factory->unTruc.strFactory = input->strFactory;
|
|
}
|
|
else
|
|
{
|
|
factory->unTruc.strFactory = NULL;
|
|
}
|
|
}
|
|
|
|
static ANTLR3_BOOLEAN
|
|
newPool(pANTLR3_TOKEN_FACTORY factory)
|
|
{
|
|
/* Increment factory count
|
|
*/
|
|
++(factory->thisPool);
|
|
|
|
// If we were reusing this token factory then we may already have a pool
|
|
// allocated. If we exceeded the max available then we must allocate a new
|
|
// one.
|
|
if (factory->thisPool > factory->maxPool)
|
|
{
|
|
/* Ensure we have enough pointers allocated
|
|
*/
|
|
pANTLR3_COMMON_TOKEN *newPools = (pANTLR3_COMMON_TOKEN *)
|
|
ANTLR3_REALLOC((void *)factory->pools, /* Current pools pointer (starts at NULL) */
|
|
(ANTLR3_UINT32)((factory->thisPool + 1) * sizeof(pANTLR3_COMMON_TOKEN *)) /* Memory for new pool pointers */
|
|
);
|
|
if (newPools == NULL)
|
|
{
|
|
// We are out of memory, but the old allocation is still valid for now
|
|
--(factory->thisPool);
|
|
return ANTLR3_FALSE;
|
|
}
|
|
|
|
factory->pools = newPools;
|
|
|
|
/* Allocate a new pool for the factory
|
|
*/
|
|
factory->pools[factory->thisPool] =
|
|
(pANTLR3_COMMON_TOKEN)
|
|
ANTLR3_CALLOC(1, (size_t)(sizeof(ANTLR3_COMMON_TOKEN) * ANTLR3_FACTORY_POOL_SIZE));
|
|
if (factory->pools[factory->thisPool] == NULL)
|
|
{
|
|
// Allocation failed
|
|
--(factory->thisPool);
|
|
return ANTLR3_FALSE;
|
|
}
|
|
|
|
// We now have a new pool and can track it as the maximum we have created so far
|
|
//
|
|
factory->maxPool = factory->thisPool;
|
|
}
|
|
|
|
/* Reset the counters
|
|
*/
|
|
factory->nextToken = 0;
|
|
|
|
/* Done
|
|
*/
|
|
return ANTLR3_TRUE;
|
|
}
|
|
|
|
static pANTLR3_COMMON_TOKEN
|
|
newPoolToken(pANTLR3_TOKEN_FACTORY factory)
|
|
{
|
|
pANTLR3_COMMON_TOKEN token;
|
|
|
|
if (factory == NULL) { return NULL; }
|
|
|
|
/* See if we need a new token pool before allocating a new
|
|
* one
|
|
*/
|
|
if (factory->nextToken >= ANTLR3_FACTORY_POOL_SIZE)
|
|
{
|
|
/* We ran out of tokens in the current pool, so we need a new pool
|
|
*/
|
|
if (!newPool(factory))
|
|
{
|
|
return NULL;
|
|
}
|
|
}
|
|
|
|
// make sure the factory is sane
|
|
if (factory->pools == NULL) { return NULL; }
|
|
if (factory->pools[factory->thisPool] == NULL) { return NULL; }
|
|
|
|
/* Assuming everything went well (we are trying for performance here so doing minimal
|
|
* error checking. Then we can work out what the pointer is to the next token.
|
|
*/
|
|
token = factory->pools[factory->thisPool] + factory->nextToken;
|
|
factory->nextToken++;
|
|
|
|
/* We have our token pointer now, so we can initialize it to the predefined model.
|
|
* We only need do this though if the token is not already initialized, we just check
|
|
* an api function pointer for this as they are allocated via calloc.
|
|
*/
|
|
if (token->setStartIndex == NULL)
|
|
{
|
|
antlr3SetTokenAPI(token);
|
|
|
|
// It is factory made, and we need to copy the string factory pointer
|
|
//
|
|
token->factoryMade = ANTLR3_TRUE;
|
|
token->strFactory = factory->input == NULL ? NULL : factory->input->strFactory;
|
|
token->input = factory->input;
|
|
}
|
|
|
|
/* And we are done
|
|
*/
|
|
return token;
|
|
}
|
|
|
|
static void
|
|
factoryReset (pANTLR3_TOKEN_FACTORY factory)
|
|
{
|
|
// Just start again with pool #0 when we are
|
|
// called.
|
|
//
|
|
factory->thisPool = -1;
|
|
newPool(factory);
|
|
}
|
|
|
|
static void
|
|
factoryClose (pANTLR3_TOKEN_FACTORY factory)
|
|
{
|
|
pANTLR3_COMMON_TOKEN pool;
|
|
ANTLR3_INT32 poolCount;
|
|
ANTLR3_UINT32 limit;
|
|
ANTLR3_UINT32 token;
|
|
pANTLR3_COMMON_TOKEN check;
|
|
|
|
/* We iterate the token pools one at a time
|
|
*/
|
|
for (poolCount = 0; poolCount <= factory->thisPool; poolCount++)
|
|
{
|
|
/* Pointer to current pool
|
|
*/
|
|
pool = factory->pools[poolCount];
|
|
|
|
/* Work out how many tokens we need to check in this pool.
|
|
*/
|
|
limit = (poolCount == factory->thisPool ? factory->nextToken : ANTLR3_FACTORY_POOL_SIZE);
|
|
|
|
/* Marginal condition, we might be at the start of a brand new pool
|
|
* where the nextToken is 0 and nothing has been allocated.
|
|
*/
|
|
if (limit > 0)
|
|
{
|
|
/* We have some tokens allocated from this pool
|
|
*/
|
|
for (token = 0; token < limit; token++)
|
|
{
|
|
/* Next one in the chain
|
|
*/
|
|
check = pool + token;
|
|
|
|
/* If the programmer made this a custom token, then
|
|
* see if we need to call their free routine.
|
|
*/
|
|
if (check->custom != NULL && check->freeCustom != NULL)
|
|
{
|
|
check->freeCustom(check->custom);
|
|
check->custom = NULL;
|
|
}
|
|
}
|
|
}
|
|
|
|
/* We can now free this pool allocation
|
|
*/
|
|
ANTLR3_FREE(factory->pools[poolCount]);
|
|
factory->pools[poolCount] = NULL;
|
|
}
|
|
|
|
/* All the pools are deallocated we can free the pointers to the pools
|
|
* now.
|
|
*/
|
|
ANTLR3_FREE(factory->pools);
|
|
|
|
/* Finally, we can free the space for the factory itself
|
|
*/
|
|
ANTLR3_FREE(factory);
|
|
}
|
|
|
|
|
|
static pANTLR3_COMMON_TOKEN
|
|
newToken(void)
|
|
{
|
|
pANTLR3_COMMON_TOKEN token;
|
|
|
|
/* Allocate memory for this
|
|
*/
|
|
token = (pANTLR3_COMMON_TOKEN) ANTLR3_CALLOC(1, (size_t)(sizeof(ANTLR3_COMMON_TOKEN)));
|
|
|
|
if (token == NULL)
|
|
{
|
|
return NULL;
|
|
}
|
|
|
|
// Install the API
|
|
//
|
|
antlr3SetTokenAPI(token);
|
|
token->factoryMade = ANTLR3_FALSE;
|
|
|
|
return token;
|
|
}
|
|
|
|
ANTLR3_API void
|
|
antlr3SetTokenAPI(pANTLR3_COMMON_TOKEN token)
|
|
{
|
|
token->getText = getText;
|
|
token->setText = setText;
|
|
token->setText8 = setText8;
|
|
token->getType = getType;
|
|
token->setType = setType;
|
|
token->getLine = getLine;
|
|
token->setLine = setLine;
|
|
token->setLine = setLine;
|
|
token->getCharPositionInLine = getCharPositionInLine;
|
|
token->setCharPositionInLine = setCharPositionInLine;
|
|
token->getChannel = getChannel;
|
|
token->setChannel = setChannel;
|
|
token->getTokenIndex = getTokenIndex;
|
|
token->setTokenIndex = setTokenIndex;
|
|
token->getStartIndex = getStartIndex;
|
|
token->setStartIndex = setStartIndex;
|
|
token->getStopIndex = getStopIndex;
|
|
token->setStopIndex = setStopIndex;
|
|
token->toString = toString;
|
|
|
|
return;
|
|
}
|
|
|
|
static pANTLR3_STRING getText (pANTLR3_COMMON_TOKEN token)
|
|
{
|
|
switch (token->textState)
|
|
{
|
|
case ANTLR3_TEXT_STRING:
|
|
|
|
// Someone already created a string for this token, so we just
|
|
// use it.
|
|
//
|
|
return token->tokText.text;
|
|
break;
|
|
|
|
case ANTLR3_TEXT_CHARP:
|
|
|
|
// We had a straight text pointer installed, now we
|
|
// must convert it to a string. Note we have to do this here
|
|
// or otherwise setText8() will just install the same char*
|
|
//
|
|
if (token->strFactory != NULL)
|
|
{
|
|
token->tokText.text = token->strFactory->newStr8(token->strFactory, (pANTLR3_UINT8)token->tokText.chars);
|
|
token->textState = ANTLR3_TEXT_STRING;
|
|
return token->tokText.text;
|
|
}
|
|
else
|
|
{
|
|
// We cannot do anything here
|
|
//
|
|
return NULL;
|
|
}
|
|
break;
|
|
|
|
default:
|
|
|
|
// EOF is a special case
|
|
//
|
|
if (token->type == ANTLR3_TOKEN_EOF)
|
|
{
|
|
token->tokText.text = token->strFactory->newStr8(token->strFactory, (pANTLR3_UINT8)"<EOF>");
|
|
token->textState = ANTLR3_TEXT_STRING;
|
|
token->tokText.text->factory = token->strFactory;
|
|
return token->tokText.text;
|
|
}
|
|
|
|
|
|
// We had nothing installed in the token, create a new string
|
|
// from the input stream
|
|
//
|
|
|
|
if (token->input != NULL)
|
|
{
|
|
|
|
return token->input->substr( token->input,
|
|
token->getStartIndex(token),
|
|
token->getStopIndex(token)
|
|
);
|
|
}
|
|
|
|
// Nothing to return, there is no input stream
|
|
//
|
|
return NULL;
|
|
break;
|
|
}
|
|
}
|
|
static void setText8 (pANTLR3_COMMON_TOKEN token, pANTLR3_UINT8 text)
|
|
{
|
|
// No text to set, so ignore
|
|
//
|
|
if (text == NULL) return;
|
|
|
|
switch (token->textState)
|
|
{
|
|
case ANTLR3_TEXT_NONE:
|
|
case ANTLR3_TEXT_CHARP: // Caller must free before setting again, if it needs to be freed
|
|
|
|
// Nothing in there yet, or just a char *, so just set the
|
|
// text as a pointer
|
|
//
|
|
token->textState = ANTLR3_TEXT_CHARP;
|
|
token->tokText.chars = (pANTLR3_UCHAR)text;
|
|
break;
|
|
|
|
default:
|
|
|
|
// It was already a pANTLR3_STRING, so just override it
|
|
//
|
|
token->tokText.text->set8(token->tokText.text, (const char *)text);
|
|
break;
|
|
}
|
|
|
|
// We are done
|
|
//
|
|
return;
|
|
}
|
|
|
|
/** \brief Install the supplied text string as teh text for the token.
|
|
* The method assumes that the existing text (if any) was created by a factory
|
|
* and so does not attempt to release any memory it is using.Text not created
|
|
* by a string fctory (not advised) should be released prior to this call.
|
|
*/
|
|
static void setText (pANTLR3_COMMON_TOKEN token, pANTLR3_STRING text)
|
|
{
|
|
// Merely replaces and existing pre-defined text with the supplied
|
|
// string
|
|
//
|
|
token->textState = ANTLR3_TEXT_STRING;
|
|
token->tokText.text = text;
|
|
|
|
/* We are done
|
|
*/
|
|
return;
|
|
}
|
|
|
|
static ANTLR3_UINT32 getType (pANTLR3_COMMON_TOKEN token)
|
|
{
|
|
return token->type;
|
|
}
|
|
|
|
static void setType (pANTLR3_COMMON_TOKEN token, ANTLR3_UINT32 type)
|
|
{
|
|
token->type = type;
|
|
}
|
|
|
|
static ANTLR3_UINT32 getLine (pANTLR3_COMMON_TOKEN token)
|
|
{
|
|
return token->line;
|
|
}
|
|
|
|
static void setLine (pANTLR3_COMMON_TOKEN token, ANTLR3_UINT32 line)
|
|
{
|
|
token->line = line;
|
|
}
|
|
|
|
static ANTLR3_INT32 getCharPositionInLine (pANTLR3_COMMON_TOKEN token)
|
|
{
|
|
return token->charPosition;
|
|
}
|
|
|
|
static void setCharPositionInLine (pANTLR3_COMMON_TOKEN token, ANTLR3_INT32 pos)
|
|
{
|
|
token->charPosition = pos;
|
|
}
|
|
|
|
static ANTLR3_UINT32 getChannel (pANTLR3_COMMON_TOKEN token)
|
|
{
|
|
return token->channel;
|
|
}
|
|
|
|
static void setChannel (pANTLR3_COMMON_TOKEN token, ANTLR3_UINT32 channel)
|
|
{
|
|
token->channel = channel;
|
|
}
|
|
|
|
static ANTLR3_MARKER getTokenIndex (pANTLR3_COMMON_TOKEN token)
|
|
{
|
|
return token->index;
|
|
}
|
|
|
|
static void setTokenIndex (pANTLR3_COMMON_TOKEN token, ANTLR3_MARKER index)
|
|
{
|
|
token->index = index;
|
|
}
|
|
|
|
static ANTLR3_MARKER getStartIndex (pANTLR3_COMMON_TOKEN token)
|
|
{
|
|
return token->start == -1 ? (ANTLR3_MARKER)(token->input->data) : token->start;
|
|
}
|
|
|
|
static void setStartIndex (pANTLR3_COMMON_TOKEN token, ANTLR3_MARKER start)
|
|
{
|
|
token->start = start;
|
|
}
|
|
|
|
static ANTLR3_MARKER getStopIndex (pANTLR3_COMMON_TOKEN token)
|
|
{
|
|
return token->stop;
|
|
}
|
|
|
|
static void setStopIndex (pANTLR3_COMMON_TOKEN token, ANTLR3_MARKER stop)
|
|
{
|
|
token->stop = stop;
|
|
}
|
|
|
|
static pANTLR3_STRING toString (pANTLR3_COMMON_TOKEN token)
|
|
{
|
|
pANTLR3_STRING text;
|
|
pANTLR3_STRING outtext;
|
|
|
|
text = token->getText(token);
|
|
|
|
if (text == NULL)
|
|
{
|
|
return NULL;
|
|
}
|
|
|
|
if (text->factory == NULL)
|
|
{
|
|
return text; // This usally means it is the EOF token
|
|
}
|
|
|
|
/* A new empty string to assemble all the stuff in
|
|
*/
|
|
outtext = text->factory->newRaw(text->factory);
|
|
|
|
/* Now we use our handy dandy string utility to assemble the
|
|
* the reporting string
|
|
* return "[@"+getTokenIndex()+","+start+":"+stop+"='"+txt+"',<"+type+">"+channelStr+","+line+":"+getCharPositionInLine()+"]";
|
|
*/
|
|
outtext->append8(outtext, "[Index: ");
|
|
outtext->addi (outtext, (ANTLR3_INT32)token->getTokenIndex(token));
|
|
outtext->append8(outtext, " (Start: ");
|
|
outtext->addi (outtext, (ANTLR3_INT32)token->getStartIndex(token));
|
|
outtext->append8(outtext, "-Stop: ");
|
|
outtext->addi (outtext, (ANTLR3_INT32)token->getStopIndex(token));
|
|
outtext->append8(outtext, ") ='");
|
|
outtext->appendS(outtext, text);
|
|
outtext->append8(outtext, "', type<");
|
|
outtext->addi (outtext, token->type);
|
|
outtext->append8(outtext, "> ");
|
|
|
|
if (token->getChannel(token) > ANTLR3_TOKEN_DEFAULT_CHANNEL)
|
|
{
|
|
outtext->append8(outtext, "(channel = ");
|
|
outtext->addi (outtext, (ANTLR3_INT32)token->getChannel(token));
|
|
outtext->append8(outtext, ") ");
|
|
}
|
|
|
|
outtext->append8(outtext, "Line: ");
|
|
outtext->addi (outtext, (ANTLR3_INT32)token->getLine(token));
|
|
outtext->append8(outtext, " LinePos:");
|
|
outtext->addi (outtext, token->getCharPositionInLine(token));
|
|
outtext->addc (outtext, ']');
|
|
|
|
return outtext;
|
|
}
|
|
|