~drizzle-trunk/drizzle/development

/* Copyright (C) 2000-2006 MySQL AB

   This program is free software; you can redistribute it and/or modify

   it under the terms of the GNU General Public License as published by

   the Free Software Foundation; version 2 of the License.

   This program is distributed in the hope that it will be useful,

   but WITHOUT ANY WARRANTY; without even the implied warranty of

   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

   GNU General Public License for more details.

   You should have received a copy of the GNU General Public License

   along with this program; if not, write to the Free Software

   Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA */

/* A lexical scanner on a temporary buffer with a yacc interface */

#define DRIZZLE_LEX 1

#include <drizzled/server_includes.h>

static int lex_one_token(void *arg, void *yythd);

/*

  We are using pointer to this variable for distinguishing between assignment

  to NEW row field (when parsing trigger definition) and structured variable.

*/

sys_var *trg_new_row_fake_var= (sys_var*) 0x01;

/**

  LEX_STRING constant for null-string to be used in parser and other places.

*/

const LEX_STRING null_lex_str= {NULL, 0};

/* Longest standard keyword name */

#define TOCK_NAME_LENGTH 24

/*

  The following data is based on the latin1 character set, and is only

  used when comparing keywords

*/

static uchar to_upper_lex[]=

{

    0,  1,  2,  3,  4,  5,  6,  7,  8,  9, 10, 11, 12, 13, 14, 15,

   16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31,

   32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47,

   48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63,

   64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79,

   80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95,

   96, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79,

   80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90,123,124,125,126,127,

  128,129,130,131,132,133,134,135,136,137,138,139,140,141,142,143,

  144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,

  160,161,162,163,164,165,166,167,168,169,170,171,172,173,174,175,

  176,177,178,179,180,181,182,183,184,185,186,187,188,189,190,191,

  192,193,194,195,196,197,198,199,200,201,202,203,204,205,206,207,

  208,209,210,211,212,213,214,215,216,217,218,219,220,221,222,223,

  192,193,194,195,196,197,198,199,200,201,202,203,204,205,206,207,

  208,209,210,211,212,213,214,247,216,217,218,219,220,221,222,255

};

/* 

  Names of the index hints (for error messages). Keep in sync with 

  index_hint_type 

*/

const char * index_hint_type_name[] =

{

  "IGNORE INDEX", 

  "USE INDEX", 

  "FORCE INDEX"

};

inline int lex_casecmp(const char *s, const char *t, uint len)

{

  while (len-- != 0 &&

	 to_upper_lex[(uchar) *s++] == to_upper_lex[(uchar) *t++]) ;

  return (int) len+1;

}

#include <lex_hash.h>

void lex_init(void)

{

  uint i;

  for (i=0 ; i < array_elements(symbols) ; i++)

    symbols[i].length=(uchar) strlen(symbols[i].name);

  for (i=0 ; i < array_elements(sql_functions) ; i++)

    sql_functions[i].length=(uchar) strlen(sql_functions[i].name);

  return;

}

void lex_free(void)

{					// Call this when daemon ends

  return;

}

void

st_parsing_options::reset()

{

  allows_variable= true;

  allows_select_into= true;

  allows_select_procedure= true;

  allows_derived= true;

}

Lex_input_stream::Lex_input_stream(THD *thd,

                                   const char* buffer,

                                   unsigned int length)

: m_thd(thd),

  yylineno(1),

  yytoklen(0),

  yylval(NULL),

  lookahead_token(END_OF_INPUT),

  lookahead_yylval(NULL),

  m_ptr(buffer),

  m_tok_start(NULL),

  m_tok_end(NULL),

  m_end_of_query(buffer + length),

  m_tok_start_prev(NULL),

  m_buf(buffer),

  m_buf_length(length),

  m_echo(true),

  m_cpp_tok_start(NULL),

  m_cpp_tok_start_prev(NULL),

  m_cpp_tok_end(NULL),

  m_body_utf8(NULL),

  m_cpp_utf8_processed_ptr(NULL),

  next_state(MY_LEX_START),

  found_semicolon(NULL),

  ignore_space(1),

  in_comment(NO_COMMENT),

  m_underscore_cs(NULL)

{

  m_cpp_buf= (char*) thd->alloc(length + 1);

  m_cpp_ptr= m_cpp_buf;

}

Lex_input_stream::~Lex_input_stream()

{}

/**

  The operation is called from the parser in order to

  1) designate the intention to have utf8 body;

  1) Indicate to the lexer that we will need a utf8 representation of this

     statement;

  2) Determine the beginning of the body.

  @param thd        Thread context.

  @param begin_ptr  Pointer to the start of the body in the pre-processed

                    buffer.

*/

void Lex_input_stream::body_utf8_start(THD *thd, const char *begin_ptr)

{

  assert(begin_ptr);

  assert(m_cpp_buf <= begin_ptr && begin_ptr <= m_cpp_buf + m_buf_length);

  uint body_utf8_length=

    (m_buf_length / thd->variables.character_set_client->mbminlen) *

    my_charset_utf8_bin.mbmaxlen;

  m_body_utf8= (char *) thd->alloc(body_utf8_length + 1);

  m_body_utf8_ptr= m_body_utf8;

  *m_body_utf8_ptr= 0;

  m_cpp_utf8_processed_ptr= begin_ptr;

}

/**

  @brief The operation appends unprocessed part of pre-processed buffer till

  the given pointer (ptr) and sets m_cpp_utf8_processed_ptr to end_ptr.

  The idea is that some tokens in the pre-processed buffer (like character

  set introducers) should be skipped.

  Example:

    CPP buffer: SELECT 'str1', _latin1 'str2';

    m_cpp_utf8_processed_ptr -- points at the "SELECT ...";

    In order to skip "_latin1", the following call should be made:

      body_utf8_append(<pointer to "_latin1 ...">, <pointer to " 'str2'...">)

  @param ptr      Pointer in the pre-processed buffer, which specifies the

                  end of the chunk, which should be appended to the utf8

                  body.

  @param end_ptr  Pointer in the pre-processed buffer, to which

                  m_cpp_utf8_processed_ptr will be set in the end of the

                  operation.

*/

void Lex_input_stream::body_utf8_append(const char *ptr,

                                        const char *end_ptr)

{

  assert(m_cpp_buf <= ptr && ptr <= m_cpp_buf + m_buf_length);

  assert(m_cpp_buf <= end_ptr && end_ptr <= m_cpp_buf + m_buf_length);

  if (!m_body_utf8)

    return;

  if (m_cpp_utf8_processed_ptr >= ptr)

    return;

  int bytes_to_copy= ptr - m_cpp_utf8_processed_ptr;

  memcpy(m_body_utf8_ptr, m_cpp_utf8_processed_ptr, bytes_to_copy);

  m_body_utf8_ptr += bytes_to_copy;

  *m_body_utf8_ptr= 0;

  m_cpp_utf8_processed_ptr= end_ptr;

}

/**

  The operation appends unprocessed part of the pre-processed buffer till

  the given pointer (ptr) and sets m_cpp_utf8_processed_ptr to ptr.

  @param ptr  Pointer in the pre-processed buffer, which specifies the end

              of the chunk, which should be appended to the utf8 body.

*/

void Lex_input_stream::body_utf8_append(const char *ptr)

{

  body_utf8_append(ptr, ptr);

}

/**

  The operation converts the specified text literal to the utf8 and appends

  the result to the utf8-body.

  @param thd      Thread context.

  @param txt      Text literal.

  @param txt_cs   Character set of the text literal.

  @param end_ptr  Pointer in the pre-processed buffer, to which

                  m_cpp_utf8_processed_ptr will be set in the end of the

                  operation.

*/

void Lex_input_stream::body_utf8_append_literal(THD *thd,

                                                const LEX_STRING *txt,

                                                const CHARSET_INFO * const txt_cs,

                                                const char *end_ptr)

{

  if (!m_cpp_utf8_processed_ptr)

    return;

  LEX_STRING utf_txt;

  if (!my_charset_same(txt_cs, &my_charset_utf8_general_ci))

  {

    thd->convert_string(&utf_txt,

                        &my_charset_utf8_general_ci,

                        txt->str, txt->length,

                        txt_cs);

  }

  else

  {

    utf_txt.str= txt->str;

    utf_txt.length= txt->length;

  }

  /* NOTE: utf_txt.length is in bytes, not in symbols. */

  memcpy(m_body_utf8_ptr, utf_txt.str, utf_txt.length);

  m_body_utf8_ptr += utf_txt.length;

  *m_body_utf8_ptr= 0;

  m_cpp_utf8_processed_ptr= end_ptr;

}

/*

  This is called before every query that is to be parsed.

  Because of this, it's critical to not do too much things here.

  (We already do too much here)

*/

void lex_start(THD *thd)

{

  LEX *lex= thd->lex;

  lex->thd= lex->unit.thd= thd;

  lex->context_stack.empty();

  lex->unit.init_query();

  lex->unit.init_select();

  /* 'parent_lex' is used in init_query() so it must be before it. */

  lex->select_lex.parent_lex= lex;

  lex->select_lex.init_query();

  lex->value_list.empty();

  lex->update_list.empty();

  lex->param_list.empty();

  lex->auxiliary_table_list.empty();

  lex->unit.next= lex->unit.master=

    lex->unit.link_next= lex->unit.return_to= 0;

  lex->unit.prev= lex->unit.link_prev= 0;

  lex->unit.slave= lex->unit.global_parameters= lex->current_select=

    lex->all_selects_list= &lex->select_lex;

  lex->select_lex.master= &lex->unit;

  lex->select_lex.prev= &lex->unit.slave;

  lex->select_lex.link_next= lex->select_lex.slave= lex->select_lex.next= 0;

  lex->select_lex.link_prev= (st_select_lex_node**)&(lex->all_selects_list);

  lex->select_lex.options= 0;

  lex->select_lex.init_order();

  lex->select_lex.group_list.empty();

  lex->describe= 0;

  lex->subqueries= false;

  lex->derived_tables= 0;

  lex->lock_option= TL_READ;

  lex->leaf_tables_insert= 0;

  lex->parsing_options.reset();

  lex->empty_field_list_on_rset= 0;

  lex->select_lex.select_number= 1;

  lex->length=0;

  lex->select_lex.in_sum_expr=0;

  lex->select_lex.ftfunc_list_alloc.empty();

  lex->select_lex.ftfunc_list= &lex->select_lex.ftfunc_list_alloc;

  lex->select_lex.group_list.empty();

  lex->select_lex.order_list.empty();

  lex->sql_command= SQLCOM_END;

  lex->duplicates= DUP_ERROR;

  lex->ignore= 0;

  lex->proc_list.first= 0;

  lex->escape_used= false;

  lex->query_tables= 0;

  lex->reset_query_tables_list(false);

  lex->expr_allows_subselect= true;

  lex->use_only_table_context= false;

  lex->name.str= 0;

  lex->name.length= 0;

  lex->nest_level=0 ;

  lex->allow_sum_func= 0;

  lex->in_sum_func= NULL;

  /*

    ok, there must be a better solution for this, long-term

    I tried "memset" in the sql_yacc.yy code, but that for

    some reason made the values zero, even if they were set

  */

  lex->server_options.server_name= 0;

  lex->server_options.server_name_length= 0;

  lex->server_options.host= 0;

  lex->server_options.db= 0;

  lex->server_options.username= 0;

  lex->server_options.password= 0;

  lex->server_options.scheme= 0;

  lex->server_options.owner= 0;

  lex->server_options.port= -1;

  lex->is_lex_started= true;

  return;

}

void lex_end(LEX *lex)

{

  if (lex->yacc_yyss)

  {

    my_free(lex->yacc_yyss, MYF(0));

    my_free(lex->yacc_yyvs, MYF(0));

    lex->yacc_yyss= 0;

    lex->yacc_yyvs= 0;

  }

  /* release used plugins */

  plugin_unlock_list(0, (plugin_ref*)lex->plugins.buffer, 

                     lex->plugins.elements);

  reset_dynamic(&lex->plugins);

  return;

}

static int find_keyword(Lex_input_stream *lip, uint len, bool function)

{

  const char *tok= lip->get_tok_start();

  SYMBOL *symbol= get_hash_symbol(tok, len, function);

  if (symbol)

  {

    lip->yylval->symbol.symbol=symbol;

    lip->yylval->symbol.str= (char*) tok;

    lip->yylval->symbol.length=len;

    return symbol->tok;

  }

  return 0;

}

/*

  Check if name is a keyword

  SYNOPSIS

    is_keyword()

    name      checked name (must not be empty)

    len       length of checked name

  RETURN VALUES

    0         name is a keyword

    1         name isn't a keyword

*/

bool is_keyword(const char *name, uint len)

{

  assert(len != 0);

  return get_hash_symbol(name,len,0)!=0;

}

bool is_lex_native_function(const LEX_STRING *name)

{

  assert(name != NULL);

  return (get_hash_symbol(name->str, name->length, 1) != 0);

}

/* make a copy of token before ptr and set yytoklen */

static LEX_STRING get_token(Lex_input_stream *lip, uint skip, uint length)

{

  LEX_STRING tmp;

  lip->yyUnget();                       // ptr points now after last token char

  tmp.length=lip->yytoklen=length;

  tmp.str= lip->m_thd->strmake(lip->get_tok_start() + skip, tmp.length);

  lip->m_cpp_text_start= lip->get_cpp_tok_start() + skip;

  lip->m_cpp_text_end= lip->m_cpp_text_start + tmp.length;

  return tmp;

}

/* 

 todo: 

   There are no dangerous charsets in mysql for function 

   get_quoted_token yet. But it should be fixed in the 

   future to operate multichar strings (like ucs2)

*/

static LEX_STRING get_quoted_token(Lex_input_stream *lip,

                                   uint skip,

                                   uint length, char quote)

{

  LEX_STRING tmp;

  const char *from, *end;

  char *to;

  lip->yyUnget();                       // ptr points now after last token char

  tmp.length= lip->yytoklen=length;

  tmp.str=(char*) lip->m_thd->alloc(tmp.length+1);

  from= lip->get_tok_start() + skip;

  to= tmp.str;

  end= to+length;

  lip->m_cpp_text_start= lip->get_cpp_tok_start() + skip;

  lip->m_cpp_text_end= lip->m_cpp_text_start + length;

  for ( ; to != end; )

  {

    if ((*to++= *from++) == quote)

    {

      from++;					// Skip double quotes

      lip->m_cpp_text_start++;

    }

  }

  *to= 0;					// End null for safety

  return tmp;

}

/*

  Return an unescaped text literal without quotes

  Fix sometimes to do only one scan of the string

*/

static char *get_text(Lex_input_stream *lip, int pre_skip, int post_skip)

{

  register uchar c,sep;

  uint found_escape=0;

  const CHARSET_INFO * const cs= lip->m_thd->charset();

  lip->tok_bitmap= 0;

  sep= lip->yyGetLast();                        // String should end with this

  while (! lip->eof())

  {

    c= lip->yyGet();

    lip->tok_bitmap|= c;

#ifdef USE_MB

    {

      int l;

      if (use_mb(cs) &&

          (l = my_ismbchar(cs,

                           lip->get_ptr() -1,

                           lip->get_end_of_query()))) {

        lip->skip_binary(l-1);

        continue;

      }

    }

#endif

    if (c == '\\')

    {					// Escaped character

      found_escape=1;

      if (lip->eof())

	return 0;

      lip->yySkip();

    }

    else if (c == sep)

    {

      if (c == lip->yyGet())            // Check if two separators in a row

      {

        found_escape=1;                 // duplicate. Remember for delete

	continue;

      }

      else

        lip->yyUnget();

      /* Found end. Unescape and return string */

      const char *str, *end;

      char *start;

      str= lip->get_tok_start();

      end= lip->get_ptr();

      /* Extract the text from the token */

      str += pre_skip;

      end -= post_skip;

      assert(end >= str);

      if (!(start= (char*) lip->m_thd->alloc((uint) (end-str)+1)))

	return (char*) "";		// Sql_alloc has set error flag

      lip->m_cpp_text_start= lip->get_cpp_tok_start() + pre_skip;

      lip->m_cpp_text_end= lip->get_cpp_ptr() - post_skip;

      if (!found_escape)

      {

	lip->yytoklen=(uint) (end-str);

	memcpy(start,str,lip->yytoklen);

	start[lip->yytoklen]=0;

      }

      else

      {

        char *to;

	for (to=start ; str != end ; str++)

	{

#ifdef USE_MB

	  int l;

	  if (use_mb(cs) &&

              (l = my_ismbchar(cs, str, end))) {

	      while (l--)

		  *to++ = *str++;

	      str--;

	      continue;

	  }

#endif

	  if (*str == '\\' && str+1 != end)

	  {

	    switch(*++str) {

	    case 'n':

	      *to++='\n';

	      break;

	    case 't':

	      *to++= '\t';

	      break;

	    case 'r':

	      *to++ = '\r';

	      break;

	    case 'b':

	      *to++ = '\b';

	      break;

	    case '0':

	      *to++= 0;			// Ascii null

	      break;

	    case 'Z':			// ^Z must be escaped on Win32

	      *to++='\032';

	      break;

	    case '_':

	    case '%':

	      *to++= '\\';		// remember prefix for wildcard

	      /* Fall through */

	    default:

              *to++= *str;

	      break;

	    }

	  }

	  else if (*str == sep)

	    *to++= *str++;		// Two ' or "

	  else

	    *to++ = *str;

	}

	*to=0;

	lip->yytoklen=(uint) (to-start);

      }

      return start;

    }

  }

  return 0;					// unexpected end of query

}

/*

** Calc type of integer; long integer, int64_t integer or real.

** Returns smallest type that match the string.

** When using unsigned long long values the result is converted to a real

** because else they will be unexpected sign changes because all calculation

** is done with int64_t or double.

*/

static const char *long_str="2147483647";

static const uint long_len=10;

static const char *signed_long_str="-2147483648";

static const char *int64_t_str="9223372036854775807";

static const uint int64_t_len=19;

static const char *signed_int64_t_str="-9223372036854775808";

static const uint signed_int64_t_len=19;

static const char *unsigned_int64_t_str="18446744073709551615";

static const uint unsigned_int64_t_len=20;

static inline uint int_token(const char *str,uint length)

{

  if (length < long_len)			// quick normal case

    return NUM;

  bool neg=0;

  if (*str == '+')				// Remove sign and pre-zeros

  {

    str++; length--;

  }

  else if (*str == '-')

  {

    str++; length--;

    neg=1;

  }

  while (*str == '0' && length)

  {

    str++; length --;

  }

  if (length < long_len)

    return NUM;

  uint smaller,bigger;

  const char *cmp;

  if (neg)

  {

    if (length == long_len)

    {

      cmp= signed_long_str+1;

      smaller=NUM;				// If <= signed_long_str

      bigger=LONG_NUM;				// If >= signed_long_str

    }

    else if (length < signed_int64_t_len)

      return LONG_NUM;

    else if (length > signed_int64_t_len)

      return DECIMAL_NUM;

    else

    {

      cmp=signed_int64_t_str+1;

      smaller=LONG_NUM;				// If <= signed_int64_t_str

      bigger=DECIMAL_NUM;

    }

  }

  else

  {

    if (length == long_len)

    {

      cmp= long_str;

      smaller=NUM;

      bigger=LONG_NUM;

    }

    else if (length < int64_t_len)

      return LONG_NUM;

    else if (length > int64_t_len)

    {

      if (length > unsigned_int64_t_len)

        return DECIMAL_NUM;

      cmp=unsigned_int64_t_str;

      smaller=ULONGLONG_NUM;

      bigger=DECIMAL_NUM;

    }

    else

    {

      cmp=int64_t_str;

      smaller=LONG_NUM;

      bigger= ULONGLONG_NUM;

    }

  }

  while (*cmp && *cmp++ == *str++) ;

  return ((uchar) str[-1] <= (uchar) cmp[-1]) ? smaller : bigger;

}

/*

  MYSQLlex remember the following states from the following MYSQLlex()

  - MY_LEX_EOQ			Found end of query

  - MY_LEX_OPERATOR_OR_IDENT	Last state was an ident, text or number

				(which can't be followed by a signed number)

*/

int MYSQLlex(void *arg, void *yythd)

{

  THD *thd= (THD *)yythd;

  Lex_input_stream *lip= thd->m_lip;

  YYSTYPE *yylval=(YYSTYPE*) arg;

  int token;

  if (lip->lookahead_token != END_OF_INPUT)

  {

    /*

      The next token was already parsed in advance,

      return it.

    */

    token= lip->lookahead_token;

    lip->lookahead_token= END_OF_INPUT;

    *yylval= *(lip->lookahead_yylval);

    lip->lookahead_yylval= NULL;

    return token;

  }

  token= lex_one_token(arg, yythd);

  switch(token) {

  case WITH:

    /*

      Parsing 'WITH' 'ROLLUP' or 'WITH' 'CUBE' requires 2 look ups,

      which makes the grammar LALR(2).

      Replace by a single 'WITH_ROLLUP' or 'WITH_CUBE' token,

      to transform the grammar into a LALR(1) grammar,

      which sql_yacc.yy can process.

    */

    token= lex_one_token(arg, yythd);

    switch(token) {

    case CUBE_SYM: