~drizzle-trunk/drizzle/development

/* Copyright (C) 2003 MySQL AB

   Copyright (C) 2010 Brian Aker

  This program is free software; you can redistribute it and/or modify

  it under the terms of the GNU General Public License as published by

  the Free Software Foundation; version 2 of the License.

  This program is distributed in the hope that it will be useful,

  but WITHOUT ANY WARRANTY; without even the implied warranty of

  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

  GNU General Public License for more details.

  You should have received a copy of the GNU General Public License

  along with this program; if not, write to the Free Software

  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA */

#include "config.h"

#include "plugin/archive/archive_engine.h"

#include <memory>

#include <boost/scoped_ptr.hpp>

using namespace std;

using namespace drizzled;

/*

  First, if you want to understand storage engines you should look at

  ha_example.cc and ha_example.h.

  This example was written as a test case for a customer who needed

  a storage engine without indexes that could compress data very well.

  So, welcome to a completely compressed storage engine. This storage

  engine only does inserts. No replace, deletes, or updates. All reads are

  complete table scans. Compression is done through a combination of packing

  and making use of the zlib library

  We keep a file pointer open for each instance of ha_archive for each read

  but for writes we keep one open file handle just for that. We flush it

  only if we have a read occur. azip handles compressing lots of records

  at once much better then doing lots of little records between writes.

  It is possible to not lock on writes but this would then mean we couldn't

  handle bulk inserts as well (that is if someone was trying to read at

  the same time since we would want to flush).

  A "meta" file is kept alongside the data file. This file serves two purpose.

  The first purpose is to track the number of rows in the table. The second

  purpose is to determine if the table was closed properly or not. When the

  meta file is first opened it is marked as dirty. It is opened when the table

  itself is opened for writing. When the table is closed the new count for rows

  is written to the meta file and the file is marked as clean. If the meta file

  is opened and it is marked as dirty, it is assumed that a crash occured. At

  this point an error occurs and the user is told to rebuild the file.

  A rebuild scans the rows and rewrites the meta file. If corruption is found

  in the data file then the meta file is not repaired.

  At some point a recovery method for such a drastic case needs to be divised.

  Locks are row level, and you will get a consistant read.

  For performance as far as table scans go it is quite fast. I don't have

  good numbers but locally it has out performed both Innodb and MyISAM. For

  Innodb the question will be if the table can be fit into the buffer

  pool. For MyISAM its a question of how much the file system caches the

  MyISAM file. With enough free memory MyISAM is faster. Its only when the OS

  doesn't have enough memory to cache entire table that archive turns out

  to be any faster.

  Examples between MyISAM (packed) and Archive.

  Table with 76695844 identical rows:

  29680807 a_archive.ARZ

  920350317 a.MYD

  Table with 8991478 rows (all of Slashdot's comments):

  1922964506 comment_archive.ARZ

  2944970297 comment_text.MYD

  TODO:

   Allow users to set compression level.

   Allow adjustable block size.

   Implement versioning, should be easy.

   Allow for errors, find a way to mark bad rows.

   Add optional feature so that rows can be flushed at interval (which will cause less

     compression but may speed up ordered searches).

   Checkpoint the meta file to allow for faster rebuilds.

   Option to allow for dirty reads, this would lower the sync calls, which would make

     inserts a lot faster, but would mean highly arbitrary reads.

    -Brian

*/

/* When the engine starts up set the first version */

static uint64_t global_version= 1;

// We use this to find out the state of the archive aio option.

extern bool archive_aio_state(void);

/*

  Number of rows that will force a bulk insert.

*/

#define ARCHIVE_MIN_ROWS_TO_USE_BULK_INSERT 2

/*

  Size of header used for row

*/

#define ARCHIVE_ROW_HEADER_SIZE 4

ArchiveShare *ArchiveEngine::findOpenTable(const string table_name)

{

  ArchiveMap::iterator find_iter=

    archive_open_tables.find(table_name);

  if (find_iter != archive_open_tables.end())

    return (*find_iter).second;

  else

    return NULL;

}

void ArchiveEngine::addOpenTable(const string &table_name, ArchiveShare *share)

{

  archive_open_tables[table_name]= share;

}

void ArchiveEngine::deleteOpenTable(const string &table_name)

{

  archive_open_tables.erase(table_name);

}

int ArchiveEngine::doDropTable(Session&, const TableIdentifier &identifier)

{

  string new_path(identifier.getPath());

  new_path+= ARZ;

  int error= unlink(new_path.c_str());

  if (error != 0)

  {

    error= errno= errno;

  }

  return error;

}

int ArchiveEngine::doGetTableDefinition(Session&,

                                        const TableIdentifier &identifier,

                                        drizzled::message::Table &table_proto)

{

  struct stat stat_info;

  int error= ENOENT;

  string proto_path;

  proto_path.reserve(FN_REFLEN);

  proto_path.assign(identifier.getPath());

  proto_path.append(ARZ);

  if (stat(proto_path.c_str(),&stat_info))

    return errno;

  else

    error= EEXIST;

  {

    boost::scoped_ptr<azio_stream> proto_stream(new azio_stream);

    char* proto_string;

    if (azopen(proto_stream.get(), proto_path.c_str(), O_RDONLY, AZ_METHOD_BLOCK) == 0)

      return HA_ERR_CRASHED_ON_USAGE;

    proto_string= (char*)malloc(sizeof(char) * proto_stream->frm_length);

    if (proto_string == NULL)

    {

      azclose(proto_stream.get());

      return ENOMEM;

    }

    azread_frm(proto_stream.get(), proto_string);

    if (table_proto.ParseFromArray(proto_string, proto_stream->frm_length) == false)

      error= HA_ERR_CRASHED_ON_USAGE;

    azclose(proto_stream.get());

    free(proto_string);

  }

  /* We set the name from what we've asked for as in RENAME TABLE for ARCHIVE

     we do not rewrite the table proto (as it's wedged in the file header)

  */

  table_proto.set_schema(identifier.getSchemaName());

  table_proto.set_name(identifier.getTableName());

  return error;

}

ha_archive::ha_archive(drizzled::plugin::StorageEngine &engine_arg,

                       Table &table_arg)

  :Cursor(engine_arg, table_arg), delayed_insert(0), bulk_insert(0)

{

  /* Set our original buffer from pre-allocated memory */

  buffer.set((char *)byte_buffer, IO_SIZE, system_charset_info);

  /* The size of the offset value we will use for position() */

  ref_length= sizeof(internal::my_off_t);

  archive_reader_open= false;

}

/*

  This method reads the header of a datafile and returns whether or not it was successful.

*/

int ha_archive::read_data_header(azio_stream *file_to_read)

{

  if (azread_init(file_to_read) == -1)

    return(HA_ERR_CRASHED_ON_USAGE);

  if (file_to_read->version >= 3)

    return(0);

  return(1);

}

ArchiveShare::ArchiveShare():

  use_count(0), archive_write_open(false), dirty(false), crashed(false),

  mean_rec_length(0), version(0), rows_recorded(0), version_rows(0)

{

  assert(1);

}

ArchiveShare::ArchiveShare(const char *name):

  use_count(0), archive_write_open(false), dirty(false), crashed(false),

  mean_rec_length(0), version(0), rows_recorded(0), version_rows(0)

{

  memset(&archive_write, 0, sizeof(azio_stream));     /* Archive file we are working with */

  table_name.append(name);

  data_file_name.assign(table_name);

  data_file_name.append(ARZ);

  /*

    We will use this lock for rows.

  */

  pthread_mutex_init(&_mutex,MY_MUTEX_INIT_FAST);

}

ArchiveShare::~ArchiveShare()

{

  _lock.deinit();

  pthread_mutex_destroy(&_mutex);

  /*

    We need to make sure we don't reset the crashed state.

    If we open a crashed file, wee need to close it as crashed unless

    it has been repaired.

    Since we will close the data down after this, we go on and count

    the flush on close;

  */

  if (archive_write_open == true)

    (void)azclose(&archive_write);

}

bool ArchiveShare::prime(uint64_t *auto_increment)

{

  boost::scoped_ptr<azio_stream> archive_tmp(new azio_stream);

  /*

    We read the meta file, but do not mark it dirty. Since we are not

    doing a write we won't mark it dirty (and we won't open it for

    anything but reading... open it for write and we will generate null

    compression writes).

  */

  if (!(azopen(archive_tmp.get(), data_file_name.c_str(), O_RDONLY,

               AZ_METHOD_BLOCK)))

    return false;

  *auto_increment= archive_tmp->auto_increment + 1;

  rows_recorded= (ha_rows)archive_tmp->rows;

  crashed= archive_tmp->dirty;

  if (version < global_version)

  {

    version_rows= rows_recorded;

    version= global_version;

  }

  azclose(archive_tmp.get());

  return true;

}

/*

  We create the shared memory space that we will use for the open table.

  No matter what we try to get or create a share. This is so that a repair

  table operation can occur.

  See ha_example.cc for a longer description.

*/

ArchiveShare *ha_archive::get_share(const char *table_name, int *rc)

{

  ArchiveEngine *a_engine= static_cast<ArchiveEngine *>(getEngine());

  pthread_mutex_lock(&a_engine->mutex());

  share= a_engine->findOpenTable(table_name);

  if (!share)

  {

    share= new ArchiveShare(table_name);

    if (share == NULL)

    {

      pthread_mutex_unlock(&a_engine->mutex());

      *rc= HA_ERR_OUT_OF_MEM;

      return(NULL);

    }

    if (share->prime(&stats.auto_increment_value) == false)

    {

      pthread_mutex_unlock(&a_engine->mutex());

      *rc= HA_ERR_CRASHED_ON_REPAIR;

      delete share;

      return NULL;

    }

    a_engine->addOpenTable(share->table_name, share);

    thr_lock_init(&share->_lock);

  }

  share->use_count++;

  if (share->crashed)

    *rc= HA_ERR_CRASHED_ON_USAGE;

  pthread_mutex_unlock(&a_engine->mutex());

  return(share);

}

/*

  Free the share.

  See ha_example.cc for a description.

*/

int ha_archive::free_share()

{

  ArchiveEngine *a_engine= static_cast<ArchiveEngine *>(getEngine());

  pthread_mutex_lock(&a_engine->mutex());

  if (!--share->use_count)

  {

    a_engine->deleteOpenTable(share->table_name);

    delete share;

  }

  pthread_mutex_unlock(&a_engine->mutex());

  return 0;

}

int ha_archive::init_archive_writer()

{

  /*

    It is expensive to open and close the data files and since you can't have

    a gzip file that can be both read and written we keep a writer open

    that is shared amoung all open tables.

  */

  if (!(azopen(&(share->archive_write), share->data_file_name.c_str(),

               O_RDWR, AZ_METHOD_BLOCK)))

  {

    share->crashed= true;

    return(1);

  }

  share->archive_write_open= true;

  return(0);

}

/*

  No locks are required because it is associated with just one Cursor instance

*/

int ha_archive::init_archive_reader()

{

  /*

    It is expensive to open and close the data files and since you can't have

    a gzip file that can be both read and written we keep a writer open

    that is shared amoung all open tables.

  */

  if (archive_reader_open == false)

  {

    az_method method;

    if (archive_aio_state())

    {

      method= AZ_METHOD_AIO;

    }

    else

    {

      method= AZ_METHOD_BLOCK;

    }

    if (!(azopen(&archive, share->data_file_name.c_str(), O_RDONLY,

                 method)))

    {

      share->crashed= true;

      return(1);

    }

    archive_reader_open= true;

  }

  return(0);

}

/*

  When opening a file we:

  Create/get our shared structure.

  Init out lock.

  We open the file we will read from.

*/

int ha_archive::doOpen(const TableIdentifier &identifier, int , uint32_t )

{

  int rc= 0;

  share= get_share(identifier.getPath().c_str(), &rc);

  /** 

    We either fix it ourselves, or we just take it offline 

    @todo Create some documentation in the recovery tools shipped with the engine.

  */

  if (rc == HA_ERR_CRASHED_ON_USAGE)

  {

    free_share();

    rc= repair();

    return 0;

  }

  else if (rc == HA_ERR_OUT_OF_MEM)

  {

    return(rc);

  }

  assert(share);

  record_buffer.resize(getTable()->getShare()->getRecordLength() + ARCHIVE_ROW_HEADER_SIZE);

  lock.init(&share->_lock);

  return(rc);

}

// Should never be called

int ha_archive::open(const char *, int, uint32_t)

{

  assert(0);

  return -1;

}

/*

  Closes the file.

  SYNOPSIS

    close();

  IMPLEMENTATION:

  We first close this storage engines file handle to the archive and

  then remove our reference count to the table (and possibly free it

  as well).

  RETURN

    0  ok

    1  Error

*/

int ha_archive::close(void)

{

  int rc= 0;

  record_buffer.clear();

  /* First close stream */

  if (archive_reader_open == true)

  {

    if (azclose(&archive))

      rc= 1;

  }

  /* then also close share */

  rc|= free_share();

  return(rc);

}

/*

  We create our data file here. The format is pretty simple.

  You can read about the format of the data file above.

  Unlike other storage engines we do not "pack" our data. Since we

  are about to do a general compression, packing would just be a waste of

  CPU time. If the table has blobs they are written after the row in the order

  of creation.

*/

int ArchiveEngine::doCreateTable(Session &,

                                 Table& table_arg,

                                 const drizzled::TableIdentifier &identifier,

                                 drizzled::message::Table& proto)

{

  int error= 0;

  boost::scoped_ptr<azio_stream> create_stream(new azio_stream);

  uint64_t auto_increment_value;

  string serialized_proto;

  auto_increment_value= proto.options().auto_increment_value();

  for (uint32_t key= 0; key < table_arg.sizeKeys(); key++)

  {

    KeyInfo *pos= &table_arg.key_info[key];

    KeyPartInfo *key_part=     pos->key_part;

    KeyPartInfo *key_part_end= key_part + pos->key_parts;

    for (; key_part != key_part_end; key_part++)

    {

      Field *field= key_part->field;

      if (!(field->flags & AUTO_INCREMENT_FLAG))

      {

        return -1;

      }

    }

  }

  std::string named_file= identifier.getPath();

  named_file.append(ARZ);

  errno= 0;

  if (azopen(create_stream.get(), named_file.c_str(), O_CREAT|O_RDWR,

             AZ_METHOD_BLOCK) == 0)

  {

    error= errno;

    unlink(named_file.c_str());

    return(error ? error : -1);

  }

  try {

    proto.SerializeToString(&serialized_proto);

  }

  catch (...)

  {

    unlink(named_file.c_str());

    return(error ? error : -1);

  }

  if (azwrite_frm(create_stream.get(), serialized_proto.c_str(),

                  serialized_proto.length()))

  {

    unlink(named_file.c_str());

    return(error ? error : -1);

  }

  if (proto.options().has_comment())

  {

    int write_length;

    write_length= azwrite_comment(create_stream.get(),

                                  proto.options().comment().c_str(),

                                  proto.options().comment().length());

    if (write_length < 0)

    {

      error= errno;

      unlink(named_file.c_str());

      return(error ? error : -1);

    }

  }

  /*

    Yes you need to do this, because the starting value

    for the autoincrement may not be zero.

  */

  create_stream->auto_increment= auto_increment_value ?

    auto_increment_value - 1 : 0;

  if (azclose(create_stream.get()))

  {

    error= errno;

    unlink(named_file.c_str());

    return(error ? error : -1);

  }

  return(0);

}

/*

  This is where the actual row is written out.

*/

int ha_archive::real_write_row(unsigned char *buf, azio_stream *writer)

{

  off_t written;

  unsigned int r_pack_length;

  /* We pack the row for writing */

  r_pack_length= pack_row(buf);

  written= azwrite_row(writer, &record_buffer[0], r_pack_length);

  if (written != r_pack_length)

  {

    return(-1);

  }

  if (!delayed_insert || !bulk_insert)

    share->dirty= true;

  return(0);

}

/*

  Calculate max length needed for row. This includes

  the bytes required for the length in the header.

*/

uint32_t ha_archive::max_row_length(const unsigned char *)

{

  uint32_t length= (uint32_t)(getTable()->getRecordLength() + getTable()->sizeFields()*2);

  length+= ARCHIVE_ROW_HEADER_SIZE;

  uint32_t *ptr, *end;

  for (ptr= getTable()->getBlobField(), end=ptr + getTable()->sizeBlobFields();

       ptr != end ;

       ptr++)

  {

      length += 2 + ((Field_blob*)getTable()->getField(*ptr))->get_length();

  }

  return length;

}

unsigned int ha_archive::pack_row(unsigned char *record)

{

  unsigned char *ptr;

  if (fix_rec_buff(max_row_length(record)))

    return(HA_ERR_OUT_OF_MEM);

  /* Copy null bits */

  memcpy(&record_buffer[0], record, getTable()->getShare()->null_bytes);

  ptr= &record_buffer[0] + getTable()->getShare()->null_bytes;

  for (Field **field=getTable()->getFields() ; *field ; field++)

  {

    if (!((*field)->is_null()))

      ptr= (*field)->pack(ptr, record + (*field)->offset(record));

  }

  return((unsigned int) (ptr - &record_buffer[0]));

}

/*

  Look at ha_archive::open() for an explanation of the row format.

  Here we just write out the row.

  Wondering about start_bulk_insert()? We don't implement it for

  archive since it optimizes for lots of writes. The only save

  for implementing start_bulk_insert() is that we could skip

  setting dirty to true each time.

*/

int ha_archive::doInsertRecord(unsigned char *buf)

{

  int rc;

  unsigned char *read_buf= NULL;

  uint64_t temp_auto;

  unsigned char *record=  getTable()->getInsertRecord();

  if (share->crashed)

    return(HA_ERR_CRASHED_ON_USAGE);

  pthread_mutex_lock(&share->mutex());

  if (share->archive_write_open == false)

    if (init_archive_writer())

      return(HA_ERR_CRASHED_ON_USAGE);

  if (getTable()->next_number_field && record == getTable()->getInsertRecord())

  {

    update_auto_increment();

    temp_auto= getTable()->next_number_field->val_int();

    /*

      We don't support decremening auto_increment. They make the performance

      just cry.

    */

    if (temp_auto <= share->archive_write.auto_increment &&

        getTable()->getShare()->getKeyInfo(0).flags & HA_NOSAME)