~drizzle-trunk/drizzle/development

/* Copyright (C) 2003 MySQL AB

  This program is free software; you can redistribute it and/or modify

  it under the terms of the GNU General Public License as published by

  the Free Software Foundation; version 2 of the License.

  This program is distributed in the hope that it will be useful,

  but WITHOUT ANY WARRANTY; without even the implied warranty of

  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

  GNU General Public License for more details.

  You should have received a copy of the GNU General Public License

  along with this program; if not, write to the Free Software

  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA */

#include <drizzled/server_includes.h>

#include <drizzled/field.h>

#include <drizzled/field/blob.h>

#include <drizzled/field/timestamp.h>

#include <storage/myisam/myisam.h>

#include <drizzled/table.h>

#include <drizzled/session.h>

#include <stdio.h>

#include <storage/archive/ha_archive.h>

/*

  First, if you want to understand storage engines you should look at

  ha_example.cc and ha_example.h.

  This example was written as a test case for a customer who needed

  a storage engine without indexes that could compress data very well.

  So, welcome to a completely compressed storage engine. This storage

  engine only does inserts. No replace, deletes, or updates. All reads are

  complete table scans. Compression is done through a combination of packing

  and making use of the zlib library

  We keep a file pointer open for each instance of ha_archive for each read

  but for writes we keep one open file handle just for that. We flush it

  only if we have a read occur. azip handles compressing lots of records

  at once much better then doing lots of little records between writes.

  It is possible to not lock on writes but this would then mean we couldn't

  handle bulk inserts as well (that is if someone was trying to read at

  the same time since we would want to flush).

  A "meta" file is kept alongside the data file. This file serves two purpose.

  The first purpose is to track the number of rows in the table. The second

  purpose is to determine if the table was closed properly or not. When the

  meta file is first opened it is marked as dirty. It is opened when the table

  itself is opened for writing. When the table is closed the new count for rows

  is written to the meta file and the file is marked as clean. If the meta file

  is opened and it is marked as dirty, it is assumed that a crash occured. At

  this point an error occurs and the user is told to rebuild the file.

  A rebuild scans the rows and rewrites the meta file. If corruption is found

  in the data file then the meta file is not repaired.

  At some point a recovery method for such a drastic case needs to be divised.

  Locks are row level, and you will get a consistant read.

  For performance as far as table scans go it is quite fast. I don't have

  good numbers but locally it has out performed both Innodb and MyISAM. For

  Innodb the question will be if the table can be fit into the buffer

  pool. For MyISAM its a question of how much the file system caches the

  MyISAM file. With enough free memory MyISAM is faster. Its only when the OS

  doesn't have enough memory to cache entire table that archive turns out

  to be any faster.

  Examples between MyISAM (packed) and Archive.

  Table with 76695844 identical rows:

  29680807 a_archive.ARZ

  920350317 a.MYD

  Table with 8991478 rows (all of Slashdot's comments):

  1922964506 comment_archive.ARZ

  2944970297 comment_text.MYD

  TODO:

   Allow users to set compression level.

   Allow adjustable block size.

   Implement versioning, should be easy.

   Allow for errors, find a way to mark bad rows.

   Add optional feature so that rows can be flushed at interval (which will cause less

     compression but may speed up ordered searches).

   Checkpoint the meta file to allow for faster rebuilds.

   Option to allow for dirty reads, this would lower the sync calls, which would make

     inserts a lot faster, but would mean highly arbitrary reads.

    -Brian

*/

/* Variables for archive share methods */

pthread_mutex_t archive_mutex;

static HASH archive_open_tables;

static unsigned int global_version;

/* The file extension */

#define ARZ ".ARZ"               // The data file

#define ARN ".ARN"               // Files used during an optimize call

/* Static declarations for handerton */

static handler *archive_create_handler(handlerton *hton,

                                       TABLE_SHARE *table,

                                       MEM_ROOT *mem_root);

static bool archive_use_aio= false;

/*

  Number of rows that will force a bulk insert.

*/

#define ARCHIVE_MIN_ROWS_TO_USE_BULK_INSERT 2

/*

  Size of header used for row

*/

#define ARCHIVE_ROW_HEADER_SIZE 4

static handler *archive_create_handler(handlerton *hton,

                                       TABLE_SHARE *table,

                                       MEM_ROOT *mem_root)

{

  return new (mem_root) ha_archive(hton, table);

}

/*

  Used for hash table that tracks open tables.

*/

static unsigned char* archive_get_key(ARCHIVE_SHARE *share, size_t *length, bool)

{

  *length=share->table_name_length;

  return (unsigned char*) share->table_name;

}

/*

  Initialize the archive handler.

  SYNOPSIS

    archive_db_init()

    void *

  RETURN

    false       OK

    true        Error

*/

int archive_db_init(void *p)

{

  handlerton *archive_hton;

  archive_hton= (handlerton *)p;

  archive_hton->state= SHOW_OPTION_YES;

  archive_hton->create= archive_create_handler;

  archive_hton->flags= HTON_NO_FLAGS;

  /* When the engine starts up set the first version */

  global_version= 1;

  if (pthread_mutex_init(&archive_mutex, MY_MUTEX_INIT_FAST))

    goto error;

  if (hash_init(&archive_open_tables, system_charset_info, 32, 0, 0,

                (hash_get_key) archive_get_key, 0, 0))

  {

    pthread_mutex_destroy(&archive_mutex);

  }

  else

  {

    return(false);

  }

error:

  return(true);

}

/*

  Release the archive handler.

  SYNOPSIS

    archive_db_done()

    void

  RETURN

    false       OK

*/

int archive_db_done(void *)

{

  hash_free(&archive_open_tables);

  pthread_mutex_destroy(&archive_mutex);

  return 0;

}

ha_archive::ha_archive(handlerton *hton, TABLE_SHARE *table_arg)

  :handler(hton, table_arg), delayed_insert(0), bulk_insert(0)

{

  /* Set our original buffer from pre-allocated memory */

  buffer.set((char *)byte_buffer, IO_SIZE, system_charset_info);

  /* The size of the offset value we will use for position() */

  ref_length= sizeof(my_off_t);

  archive_reader_open= false;

}

/*

  This method reads the header of a datafile and returns whether or not it was successful.

*/

int ha_archive::read_data_header(azio_stream *file_to_read)

{

  if (azread_init(file_to_read) == -1)

    return(HA_ERR_CRASHED_ON_USAGE);

  if (file_to_read->version >= 3)

    return(0);

  return(1);

}

/*

  We create the shared memory space that we will use for the open table.

  No matter what we try to get or create a share. This is so that a repair

  table operation can occur.

  See ha_example.cc for a longer description.

*/

ARCHIVE_SHARE *ha_archive::get_share(const char *table_name, int *rc)

{

  uint32_t length;

  pthread_mutex_lock(&archive_mutex);

  length=(uint) strlen(table_name);

  if (!(share=(ARCHIVE_SHARE*) hash_search(&archive_open_tables,

                                           (unsigned char*) table_name,

                                           length)))

  {

    char *tmp_name;

    azio_stream archive_tmp;

    if (!my_multi_malloc(MYF(MY_WME | MY_ZEROFILL),

                          &share, sizeof(*share),

                          &tmp_name, length+1,

                          NULL))

    {

      pthread_mutex_unlock(&archive_mutex);

      *rc= HA_ERR_OUT_OF_MEM;

      return(NULL);

    }

    share->use_count= 0;

    share->table_name_length= length;

    share->table_name= tmp_name;

    share->crashed= false;

    share->archive_write_open= false;

    fn_format(share->data_file_name, table_name, "",

              ARZ, MY_REPLACE_EXT | MY_UNPACK_FILENAME);

    strcpy(share->table_name, table_name);

    /*

      We will use this lock for rows.

    */

    pthread_mutex_init(&share->mutex,MY_MUTEX_INIT_FAST);

    /*

      We read the meta file, but do not mark it dirty. Since we are not

      doing a write we won't mark it dirty (and we won't open it for

      anything but reading... open it for write and we will generate null

      compression writes).

    */

    if (!(azopen(&archive_tmp, share->data_file_name, O_RDONLY,

                 AZ_METHOD_BLOCK)))

    {

      pthread_mutex_destroy(&share->mutex);

      free(share);

      pthread_mutex_unlock(&archive_mutex);

      *rc= HA_ERR_CRASHED_ON_REPAIR;

      return(NULL);

    }

    stats.auto_increment_value= archive_tmp.auto_increment + 1;

    share->rows_recorded= (ha_rows)archive_tmp.rows;

    share->crashed= archive_tmp.dirty;

    if (share->version < global_version)

    {

      share->version_rows= share->rows_recorded;

      share->version= global_version;

    }

    azclose(&archive_tmp);

    my_hash_insert(&archive_open_tables, (unsigned char*) share);

    thr_lock_init(&share->lock);

  }

  share->use_count++;

  if (share->crashed)

    *rc= HA_ERR_CRASHED_ON_USAGE;

  pthread_mutex_unlock(&archive_mutex);

  return(share);

}

/*

  Free the share.

  See ha_example.cc for a description.

*/

int ha_archive::free_share()

{

  int rc= 0;

  pthread_mutex_lock(&archive_mutex);

  if (!--share->use_count)

  {

    hash_delete(&archive_open_tables, (unsigned char*) share);

    thr_lock_delete(&share->lock);

    pthread_mutex_destroy(&share->mutex);

    /*

      We need to make sure we don't reset the crashed state.

      If we open a crashed file, wee need to close it as crashed unless

      it has been repaired.

      Since we will close the data down after this, we go on and count

      the flush on close;

    */

    if (share->archive_write_open == true)

    {

      if (azclose(&(share->archive_write)))

        rc= 1;

    }

    free((unsigned char*) share);

  }

  pthread_mutex_unlock(&archive_mutex);

  return(rc);

}

int ha_archive::init_archive_writer()

{

  /*

    It is expensive to open and close the data files and since you can't have

    a gzip file that can be both read and written we keep a writer open

    that is shared amoung all open tables.

  */

  if (!(azopen(&(share->archive_write), share->data_file_name,

               O_RDWR, AZ_METHOD_BLOCK)))

  {

    share->crashed= true;

    return(1);

  }

  share->archive_write_open= true;

  return(0);

}

/*

  No locks are required because it is associated with just one handler instance

*/

int ha_archive::init_archive_reader()

{

  /*

    It is expensive to open and close the data files and since you can't have

    a gzip file that can be both read and written we keep a writer open

    that is shared amoung all open tables.

  */

  if (archive_reader_open == false)

  {

    az_method method;

    switch (archive_use_aio)

    {

    case false:

      method= AZ_METHOD_BLOCK;

      break;

    case true:

      method= AZ_METHOD_AIO;

      break;

    default:

      method= AZ_METHOD_BLOCK;

    }

    if (!(azopen(&archive, share->data_file_name, O_RDONLY,

                 method)))

    {

      share->crashed= true;

      return(1);

    }

    archive_reader_open= true;

  }

  return(0);

}

/*

  We just implement one additional file extension.

*/

static const char *ha_archive_exts[] = {

  ARZ,

  NULL

};

const char **ha_archive::bas_ext() const

{

  return ha_archive_exts;

}

/*

  When opening a file we:

  Create/get our shared structure.

  Init out lock.

  We open the file we will read from.

*/

int ha_archive::open(const char *name, int, uint32_t open_options)

{

  int rc= 0;

  share= get_share(name, &rc);

  if (rc == HA_ERR_CRASHED_ON_USAGE && !(open_options & HA_OPEN_FOR_REPAIR))

  {

    /* purecov: begin inspected */

    free_share();

    return(rc);

    /* purecov: end */

  }

  else if (rc == HA_ERR_OUT_OF_MEM)

  {

    return(rc);

  }

  assert(share);

  record_buffer= create_record_buffer(table->s->reclength +

                                      ARCHIVE_ROW_HEADER_SIZE);

  if (!record_buffer)

  {

    free_share();

    return(HA_ERR_OUT_OF_MEM);

  }

  thr_lock_data_init(&share->lock, &lock, NULL);

  if (rc == HA_ERR_CRASHED_ON_USAGE && open_options & HA_OPEN_FOR_REPAIR)

  {

    return(0);

  }

  else

    return(rc);

}

/*

  Closes the file.

  SYNOPSIS

    close();

  IMPLEMENTATION:

  We first close this storage engines file handle to the archive and

  then remove our reference count to the table (and possibly free it

  as well).

  RETURN

    0  ok

    1  Error

*/

int ha_archive::close(void)

{

  int rc= 0;

  destroy_record_buffer(record_buffer);

  /* First close stream */

  if (archive_reader_open == true)

  {

    if (azclose(&archive))

      rc= 1;

  }

  /* then also close share */

  rc|= free_share();

  return(rc);

}

/*

  We create our data file here. The format is pretty simple.

  You can read about the format of the data file above.

  Unlike other storage engines we do not "pack" our data. Since we

  are about to do a general compression, packing would just be a waste of

  CPU time. If the table has blobs they are written after the row in the order

  of creation.

*/

int ha_archive::create(const char *name, Table *table_arg,

                       HA_CREATE_INFO *create_info)

{

  char name_buff[FN_REFLEN];

  char linkname[FN_REFLEN];

  int error= 0;

  azio_stream create_stream;            /* Archive file we are working with */

  FILE *frm_file;                   /* File handler for readers */

  struct stat file_stat;

  unsigned char *frm_ptr;

  int r;

  stats.auto_increment_value= create_info->auto_increment_value;

  for (uint32_t key= 0; key < table_arg->sizeKeys(); key++)

  {

    KEY *pos= table_arg->key_info+key;

    KEY_PART_INFO *key_part=     pos->key_part;

    KEY_PART_INFO *key_part_end= key_part + pos->key_parts;

    for (; key_part != key_part_end; key_part++)

    {

      Field *field= key_part->field;

      if (!(field->flags & AUTO_INCREMENT_FLAG))

      {

        error= -1;

        goto error;

      }

    }

  }

  /*

    We reuse name_buff since it is available.

  */

  if (create_info->data_file_name && create_info->data_file_name[0] != '#')

  {

    fn_format(name_buff, create_info->data_file_name, "", ARZ,

              MY_REPLACE_EXT | MY_UNPACK_FILENAME);

    fn_format(linkname, name, "", ARZ,

              MY_REPLACE_EXT | MY_UNPACK_FILENAME);

  }

  else

  {

    fn_format(name_buff, name, "", ARZ,

              MY_REPLACE_EXT | MY_UNPACK_FILENAME);

    linkname[0]= 0;

  }

  /*

    There is a chance that the file was "discovered". In this case

    just use whatever file is there.

  */

  r= stat(name_buff, &file_stat);

  if (r == -1 && errno!=ENOENT)

  {

    return errno;

  }

  if (!r)

    return HA_ERR_TABLE_EXIST;

  my_errno= 0;

  if (!(azopen(&create_stream, name_buff, O_CREAT|O_RDWR,

	       AZ_METHOD_BLOCK)))

  {

    error= errno;

    goto error2;

  }

  if (linkname[0])

    my_symlink(name_buff, linkname, MYF(0));

  fn_format(name_buff, name, "", ".frm",

	    MY_REPLACE_EXT | MY_UNPACK_FILENAME);

  /*

    Here is where we open up the frm and pass it to archive to store

  */

  if ((frm_file= fopen(name_buff, "r")) > 0)

  {

    if (fstat(fileno(frm_file), &file_stat))

    {

      if ((uint64_t)file_stat.st_size > SIZE_MAX)

      {

        error= ENOMEM;

        goto error2;

      }

      frm_ptr= (unsigned char *)malloc((size_t)file_stat.st_size);

      if (frm_ptr)

      {

        size_t length_io;

	length_io= read(fileno(frm_file), frm_ptr, (size_t)file_stat.st_size);

        if (length_io != (size_t)file_stat.st_size)

        {

          free(frm_ptr);

          goto error2;

        }

	length_io= azwrite_frm(&create_stream, (char *)frm_ptr, (size_t)file_stat.st_size);

        if (length_io != (size_t)file_stat.st_size)

        {

          free(frm_ptr);

          goto error2;

        }

	free(frm_ptr);

      }

    }

    fclose(frm_file);

  }

  if (create_info->comment.str)

  {

    size_t write_length;

    write_length= azwrite_comment(&create_stream, create_info->comment.str,

                                  (unsigned int)create_info->comment.length);

    if (write_length == (size_t)create_info->comment.length)

      goto error2;

  }

  /*

    Yes you need to do this, because the starting value

    for the autoincrement may not be zero.

  */

  create_stream.auto_increment= stats.auto_increment_value ?

    stats.auto_increment_value - 1 : 0;

  if (azclose(&create_stream))

  {

    error= errno;

    goto error2;

  }

  return(0);

error2:

  delete_table(name);

error:

  /* Return error number, if we got one */

  return(error ? error : -1);

}

/*

  This is where the actual row is written out.

*/

int ha_archive::real_write_row(unsigned char *buf, azio_stream *writer)

{

  off_t written;

  unsigned int r_pack_length;

  /* We pack the row for writing */

  r_pack_length= pack_row(buf);

  written= azwrite_row(writer, record_buffer->buffer, r_pack_length);

  if (written != r_pack_length)

  {

    return(-1);

  }

  if (!delayed_insert || !bulk_insert)

    share->dirty= true;

  return(0);

}

/*

  Calculate max length needed for row. This includes

  the bytes required for the length in the header.

*/

uint32_t ha_archive::max_row_length(const unsigned char *)

{

  uint32_t length= (uint32_t)(table->getRecordLength() + table->sizeFields()*2);

  length+= ARCHIVE_ROW_HEADER_SIZE;

  uint32_t *ptr, *end;

  for (ptr= table->getBlobField(), end=ptr + table->sizeBlobFields();

       ptr != end ;

       ptr++)

  {

      length += 2 + ((Field_blob*)table->field[*ptr])->get_length();

  }

  return length;

}

unsigned int ha_archive::pack_row(unsigned char *record)

{

  unsigned char *ptr;

  if (fix_rec_buff(max_row_length(record)))

    return(HA_ERR_OUT_OF_MEM); /* purecov: inspected */

  /* Copy null bits */

  memcpy(record_buffer->buffer, record, table->s->null_bytes);

  ptr= record_buffer->buffer + table->s->null_bytes;

  for (Field **field=table->field ; *field ; field++)

  {

    if (!((*field)->is_null()))

      ptr= (*field)->pack(ptr, record + (*field)->offset(record));

  }

  return((unsigned int) (ptr - record_buffer->buffer));

}

/*

  Look at ha_archive::open() for an explanation of the row format.

  Here we just write out the row.

  Wondering about start_bulk_insert()? We don't implement it for

  archive since it optimizes for lots of writes. The only save

  for implementing start_bulk_insert() is that we could skip

  setting dirty to true each time.

*/

int ha_archive::write_row(unsigned char *buf)

{

  int rc;

  unsigned char *read_buf= NULL;

  uint64_t temp_auto;

  unsigned char *record=  table->record[0];

  if (share->crashed)

    return(HA_ERR_CRASHED_ON_USAGE);

  ha_statistic_increment(&SSV::ha_write_count);

  if (table->timestamp_field_type & TIMESTAMP_AUTO_SET_ON_INSERT)

    table->timestamp_field->set_time();

  pthread_mutex_lock(&share->mutex);

  if (share->archive_write_open == false)

    if (init_archive_writer())

      return(HA_ERR_CRASHED_ON_USAGE);

  if (table->next_number_field && record == table->record[0])

  {

    KEY *mkey= &table->s->key_info[0]; // We only support one key right now

    update_auto_increment();

    temp_auto= table->next_number_field->val_int();

    /*

      We don't support decremening auto_increment. They make the performance

      just cry.

    */

    if (temp_auto <= share->archive_write.auto_increment &&