1
/* Copyright (c) 2009 PrimeBase Technologies GmbH, Germany
3
* PrimeBase Media Stream for MySQL
5
* This program is free software; you can redistribute it and/or modify
6
* it under the terms of the GNU General Public License as published by
7
* the Free Software Foundation; either version 2 of the License, or
8
* (at your option) any later version.
10
* This program is distributed in the hope that it will be useful,
11
* but WITHOUT ANY WARRANTY; without even the implied warranty of
12
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13
* GNU General Public License for more details.
15
* You should have received a copy of the GNU General Public License
16
* along with this program; if not, write to the Free Software
17
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
19
* Created by Barry Leslie on 3/20/09.
25
#include <drizzled/common.h>
26
#include <drizzled/session.h>
27
#include <drizzled/table.h>
28
#include <drizzled/message/table.pb.h>
29
#include "drizzled/charset_info.h"
30
#include <drizzled/table_proto.h>
31
#include <drizzled/session.h>
32
#include <drizzled/field.h>
35
#include "cslib/CSConfig.h"
41
#include "cslib/CSGlobal.h"
42
#include "cslib/CSThread.h"
43
#include "cslib/CSLog.h"
44
#include "cslib/CSPath.h"
45
#include "cslib/CSFile.h"
46
#include "cslib/CSString.h"
47
#include "cslib/CSStrUtil.h"
48
#include "cslib/CSStorage.h"
49
#include "cslib/CSEncode.h"
50
#include "cslib/CSS3Protocol.h"
52
#include "backup_ms.h"
55
CSSyncSparseArray *MSCloudInfo::gCloudInfo;
56
uint32_t MSCloudInfo::gMaxInfoRef;
58
uint32_t CloudDB::gKeyIndex;
59
CSMutex CloudDB::gCloudKeyLock;
61
//==============================
62
MSCloudInfo::MSCloudInfo(uint32_t id,
64
const char *bucket_arg,
65
const char *publicKey,
66
const char *privateKey
72
new_(s3Prot, CSS3Protocol());
73
s3Prot->s3_setServer(server);
74
s3Prot->s3_setPublicKey(publicKey);
75
s3Prot->s3_setPrivateKey(privateKey);
77
bucket = CSString::newString(bucket_arg);
80
//-------------------------------
81
MSCloudInfo::~MSCloudInfo()
90
//-------------------------------
91
const char *MSCloudInfo::getServer()
93
return s3Prot->s3_getServer();
96
//-------------------------------
97
const char *MSCloudInfo::getBucket()
99
return bucket->getCString();
102
//-------------------------------
103
const char *MSCloudInfo::getPublicKey()
105
return s3Prot->s3_getPublicKey();
108
//-------------------------------
109
const char *MSCloudInfo::getPrivateKey()
111
return s3Prot->s3_getPrivateKey();
114
//-------------------------------
115
CSString *MSCloudInfo::getSignature(const char *key, const char *content_type, uint32_t *s3AuthorizationTime)
117
return s3Prot->s3_getAuthorization(bucket->getCString(), key, content_type, s3AuthorizationTime);
120
//-------------------------------
121
CSString *MSCloudInfo::getDataURL(const char *key, int keep_alive)
123
return s3Prot->s3_getDataURL(bucket->getCString(), key, keep_alive);
126
//-------------------------------
127
void MSCloudInfo::send(CSInputStream *input, const char *key, off64_t size)
130
headers = s3Prot->s3_send(input, bucket->getCString(), key, size);
134
//-------------------------------
135
CSVector *MSCloudInfo::list(const char *key_prefix, uint32_t max)
137
return s3Prot->s3_list(bucket->getCString(), key_prefix, max);
140
//-------------------------------
141
void MSCloudInfo::receive(CSOutputStream *output, const char *key)
146
headers = s3Prot->s3_receive(output, bucket->getCString(), key, &found);
152
new_(err, CSStringBuffer());
154
err->append("S3 object not found: ");
155
err->append(getServer());
157
err->append(bucket->getCString());
161
CSException::throwException(CS_CONTEXT, CS_ERR_GENERIC_ERROR, err->getCString());
167
//-------------------------------
168
void MSCloudInfo::cDelete(const char *key)
170
s3Prot->s3_delete(bucket->getCString(), key);
173
//-------------------------------
174
void MSCloudInfo::copy(MSCloudInfo *dst_cloud, const char *dst_key, const char *src_key)
179
s3Prot->s3_copy(dst_cloud->getServer() ,dst_cloud->bucket->getCString(), dst_key, bucket->getCString(), src_key);
185
//==============================
186
CloudDB::CloudDB(uint32_t db_id):
188
keep_alive(5 * 60),// default URL keep alive in seconds.
198
new_(clObjectKey, CSStringBuffer());
199
clObjectKey->setLength(base_key_size);
204
//-------------------------------
209
backupInfo->release();
212
backupCloud->release();
215
clObjectKey->release();
218
//-------------------------------
219
MSBackupInfo *CloudDB::cl_getBackupInfo()
222
backupInfo->retain();
227
//-------------------------------
228
void CloudDB::cl_clearBackupInfo(){ backupInfo->release(); backupInfo = NULL;}
230
//-------------------------------
231
void CloudDB::cl_createDB()
236
//-------------------------------
238
void CloudDB::cl_restoreDB()
240
CSVector *list = NULL;
241
CSString *key = NULL;
242
CloudObjectKey *src_objectKey = NULL, *dst_objectKey = NULL;
243
CloudKeyRec cloudKey;
244
uint32_t src_cloudRef, dst_cloudRef = 0;
245
MSBackupInfo *backup_info = NULL;
246
MSCloudInfo *src_cloud = NULL, *dst_cloud = NULL;
249
if (!blob_recovery_no)
250
exit_(); // nothing to do.
252
backup_info = MSBackupInfo::getBackupInfo(blob_recovery_no);
255
src_cloudRef = backup_info->getcloudRef();
256
src_cloud = MSCloudInfo::getCloudInfo(src_cloudRef);
259
new_(dst_objectKey, CloudObjectKey(blob_db_id));
260
push_(dst_objectKey);
262
// Get the key for the backup BLOB
263
new_(src_objectKey, CloudObjectKey(blob_db_id));
264
push_(src_objectKey);
265
src_objectKey->setObjectKey(NULL, backup_info->getcloudBackupNo(), backup_info->getDatabaseId());
267
// Get a list of all the BLOBs that were backed up.
268
list = src_cloud->list(src_objectKey->getCString());
269
release_(src_objectKey);
272
// Go through the list copying the keys.
274
dst_cloudRef = src_cloudRef;
275
dst_cloud = src_cloud;
278
while ((key = (CSString*)(list->take(0))) ) {
281
// The source key name must be parsed to get its
282
// destination cloud reference. The destination for
283
// the BLOBs may not all be in the same cloud.
284
CloudObjectKey::parseObjectKey(key->getCString(), &cloudKey);
286
// Reset the destination cloud if required.
287
if (cloudKey.cloud_ref != dst_cloudRef) {
289
dst_cloud->release();
292
dst_cloudRef = cloudKey.cloud_ref;
293
dst_cloud = MSCloudInfo::getCloudInfo(dst_cloudRef);
296
// Copy the BLOB to the recovered database.
297
dst_objectKey->setObjectKey(&cloudKey);
298
src_cloud->copy(RETAIN(dst_cloud), dst_objectKey->getCString(), key->getCString());
306
dst_cloud->release();
308
finally_end_block(a);
310
blob_recovery_no = 0;
312
release_(dst_objectKey);
314
release_(backup_info);
318
//-------------------------------
319
uint32_t CloudDB::cl_getNextBackupNumber(uint32_t cloud_ref)
321
CloudObjectKey *objectKey;
323
uint32_t backup_no = 0, size = 1;
324
MSCloudInfo *s3Cloud;
327
s3Cloud = MSCloudInfo::getCloudInfo((cloud_ref)?cloud_ref:dfltCloudRefId);
330
new_(objectKey, CloudObjectKey(blob_db_id));
333
// Find the next available backup number
336
objectKey->setObjectKey(NULL, backup_no); // use the key prefix with the backup number for listing.
337
list = s3Cloud->list(objectKey->getCString(), 1);
348
//-------------------------------
349
void CloudDB::cl_backupBLOB(CloudKeyPtr key)
351
CloudObjectKey *src_objectKey, *dst_objectKey;
352
uint32_t cloudRef, backupNo;
353
MSCloudInfo *src_cloud = NULL, *dst_cloud = NULL;
358
if ((cloudRef = backupInfo->getcloudRef()) == 0) {
359
backupInfo->setcloudRef(dfltCloudRefId);
360
cloudRef = dfltCloudRefId;
363
if ((backupNo = backupInfo->getcloudBackupNo()) == 0) {
364
backupNo = cl_getNextBackupNumber(cloudRef);
365
backupInfo->setcloudBackupNo(backupNo);
368
// Set the source object's key
369
new_(src_objectKey, CloudObjectKey(blob_db_id));
370
push_(src_objectKey);
371
src_objectKey->setObjectKey(key);
373
// Set the destination object's key
374
new_(dst_objectKey, CloudObjectKey(blob_db_id));
375
push_(dst_objectKey);
376
dst_objectKey->setObjectKey(key, backupNo);
378
// Get the source cloud
379
src_cloud = MSCloudInfo::getCloudInfo((key->cloud_ref)?key->cloud_ref:dfltCloudRefId);
382
// Copy the object to the destination cloud
383
dst_cloud = MSCloudInfo::getCloudInfo(cloudRef);
384
src_cloud->copy(dst_cloud, dst_objectKey->getCString(), src_objectKey->getCString());
387
release_(dst_objectKey);
388
release_(src_objectKey);
392
//-------------------------------
393
void CloudDB::cl_restoreBLOB(CloudKeyPtr key, uint32_t backup_db_id)
395
CloudObjectKey *src_objectKey, *dst_objectKey;
396
uint32_t cloudRef, backupNo;
397
MSCloudInfo *src_cloud = NULL, *dst_cloud = NULL;
402
if ((cloudRef = backupInfo->getcloudRef()) == 0) {
403
backupInfo->setcloudRef(dfltCloudRefId);
404
cloudRef = dfltCloudRefId;
407
if ((backupNo = backupInfo->getcloudBackupNo()) == 0) {
408
backupNo = cl_getNextBackupNumber(cloudRef);
409
backupInfo->setcloudBackupNo(backupNo);
412
// Set the source object's key
413
new_(src_objectKey, CloudObjectKey(backup_db_id));
414
push_(src_objectKey);
415
src_objectKey->setObjectKey(key, backupNo);
417
// Set the destination object's key
418
new_(dst_objectKey, CloudObjectKey(blob_db_id));
419
push_(dst_objectKey);
420
dst_objectKey->setObjectKey(key);
422
// Get the source cloud
423
src_cloud = MSCloudInfo::getCloudInfo(cloudRef);
426
// Copy the object to the destination cloud
427
dst_cloud = MSCloudInfo::getCloudInfo((key->cloud_ref)?key->cloud_ref:dfltCloudRefId);
428
src_cloud->copy(dst_cloud, dst_objectKey->getCString(), src_objectKey->getCString());
431
release_(dst_objectKey);
432
release_(src_objectKey);
436
//-------------------------------
437
// Drop database deletes all objects with the database key prefix
438
void CloudDB::cl_dropDB()
442
CloudObjectKey *objectKey;
443
MSCloudInfo *s3Cloud = NULL;
448
new_(objectKey, CloudObjectKey(blob_db_id));
451
lock_(MSCloudInfo::gCloudInfo);
455
if (backupInfo && (backup_no = backupInfo->getcloudBackupNo())) {
456
objectKey->setObjectKey(NULL, backup_no); // use the key prefix for the backup for listing.
457
if ((s3Cloud = MSCloudInfo::getCloudInfo(backupInfo->getcloudRef())))
461
objectKey->setObjectKey(); // use the key prefix for listing.
463
s3Cloud = (MSCloudInfo*)MSCloudInfo::gCloudInfo->itemAt(i++); // <-- unreferenced object
466
key_str = objectKey->getCString();
468
// For non backup BLOBs all known clouds must be searched
469
// for possible BLOBs and deleted. The BLOBs belonging to a backup
470
// will ever only be in one cloud storage location.
472
list = s3Cloud->list(key_str);
475
// Go through the list deleting the keys.
476
while ((key = (CSString*)(list->take(0))) ) {
478
s3Cloud->cDelete(key->getCString());
484
release_(s3Cloud); // Only the backup s3Cloud needs to be released.
487
s3Cloud = (MSCloudInfo*)MSCloudInfo::gCloudInfo->itemAt(i++);// <-- unreferenced object
490
unlock_(MSCloudInfo::gCloudInfo);
495
//-------------------------------
496
void CloudDB::cl_putData(CloudKeyPtr key, CSInputStream *stream, off64_t size)
498
CloudObjectKey *objectKey;
499
MSCloudInfo *s3Cloud;
505
new_(objectKey, CloudObjectKey(blob_db_id));
508
objectKey->setObjectKey(key);
510
s3Cloud = MSCloudInfo::getCloudInfo((key->cloud_ref)?key->cloud_ref:dfltCloudRefId);
512
s3Cloud->send(RETAIN(stream), objectKey->getCString(), size);
521
//-------------------------------
522
off64_t CloudDB::cl_getData(CloudKeyPtr key, char *buffer, off64_t size)
524
CloudObjectKey *objectKey;
525
CSStaticMemoryOutputStream *output;
526
MSCloudInfo *s3Cloud;
529
new_(objectKey, CloudObjectKey(blob_db_id));
532
s3Cloud = MSCloudInfo::getCloudInfo(key->cloud_ref);
535
new_(output, CSStaticMemoryOutputStream((u_char *)buffer, size));
538
objectKey->setObjectKey(key);
540
s3Cloud->receive(RETAIN(output), objectKey->getCString());
541
size = output->getSize();
549
//-------------------------------
550
void CloudDB::cl_deleteData(CloudKeyPtr key)
552
MSCloudInfo *s3Cloud;
553
CloudObjectKey *objectKey;
556
new_(objectKey, CloudObjectKey(blob_db_id));
559
s3Cloud = MSCloudInfo::getCloudInfo(key->cloud_ref);
562
objectKey->setObjectKey(key);
564
s3Cloud->cDelete(objectKey->getCString());
572
//-------------------------------
573
CSString *CloudDB::cl_getDataURL(CloudKeyPtr key)
575
CloudObjectKey *objectKey;
577
MSCloudInfo *s3Cloud;
580
new_(objectKey, CloudObjectKey(blob_db_id));
583
objectKey->setObjectKey(key);
585
s3Cloud = MSCloudInfo::getCloudInfo(key->cloud_ref);
588
url = s3Cloud->getDataURL(objectKey->getCString(), keep_alive);
596
//-------------------------------
597
CSString *CloudDB::cl_getSignature(CloudKeyPtr key, CSString *content_type_arg, uint32_t *s3AuthorizationTime)
600
CloudObjectKey *objectKey;
601
const char *content_type = NULL;
602
MSCloudInfo *s3Cloud;
605
new_(objectKey, CloudObjectKey(blob_db_id));
608
if (content_type_arg) {
609
push_(content_type_arg);
610
content_type = content_type_arg->getCString();
613
objectKey->setObjectKey(key);
614
s3Cloud = MSCloudInfo::getCloudInfo(key->cloud_ref);
617
signature = s3Cloud->getSignature(objectKey->getCString(), content_type, s3AuthorizationTime);
619
if (content_type_arg)
620
release_(content_type_arg);
628
//==============================