~drizzle-trunk/drizzle/development

« back to all changes in this revision

Viewing changes to plugin/pbms/src/cloud.cc

Added the PBMS daemon plugin.

(Augen zu und durch!)

Show diffs side-by-side

added added

removed removed

Lines of Context:
 
1
/* Copyright (c) 2009 PrimeBase Technologies GmbH, Germany
 
2
 *
 
3
 * PrimeBase Media Stream for MySQL
 
4
 *
 
5
 * This program is free software; you can redistribute it and/or modify
 
6
 * it under the terms of the GNU General Public License as published by
 
7
 * the Free Software Foundation; either version 2 of the License, or
 
8
 * (at your option) any later version.
 
9
 *
 
10
 * This program is distributed in the hope that it will be useful,
 
11
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 
12
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 
13
 * GNU General Public License for more details.
 
14
 *
 
15
 * You should have received a copy of the GNU General Public License
 
16
 * along with this program; if not, write to the Free Software
 
17
 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
 
18
 *
 
19
 *  Created by Barry Leslie on 3/20/09.
 
20
 *
 
21
 */
 
22
 
 
23
#include "CSConfig.h"
 
24
 
 
25
#include <stdlib.h>
 
26
#include <ctype.h>
 
27
#include <string.h>
 
28
 
 
29
#include "CSGlobal.h"
 
30
#include "CSThread.h"
 
31
#include "CSLog.h"
 
32
#include "CSPath.h"
 
33
#include "CSFile.h"
 
34
#include "CSString.h"
 
35
#include "CSStrUtil.h"
 
36
#include "CSStorage.h"
 
37
#include "CSEncode.h"
 
38
#include "CSS3Protocol.h"
 
39
 
 
40
#include "backup_ms.h"
 
41
#include "cloud.h"
 
42
 
 
43
CSSyncSparseArray       *MSCloudInfo::gCloudInfo;
 
44
uint32_t                        MSCloudInfo::gMaxInfoRef;
 
45
 
 
46
uint32_t                        CloudDB::gKeyIndex;
 
47
CSMutex                 CloudDB::gCloudKeyLock;
 
48
 
 
49
//==============================
 
50
MSCloudInfo::MSCloudInfo(uint32_t id,
 
51
                                                const char *server,
 
52
                                                const char *bucket_arg,
 
53
                                                const char *publicKey,
 
54
                                                const char *privateKey
 
55
                                                ):
 
56
        cloudRefId(id),
 
57
        bucket(NULL),
 
58
        s3Prot(NULL)
 
59
{               
 
60
        new_(s3Prot, CSS3Protocol());
 
61
        s3Prot->s3_setServer(server);
 
62
        s3Prot->s3_setPublicKey(publicKey);
 
63
        s3Prot->s3_setPrivateKey(privateKey);
 
64
        
 
65
        bucket = CSString::newString(bucket_arg);               
 
66
}
 
67
                
 
68
//-------------------------------
 
69
MSCloudInfo::~MSCloudInfo()
 
70
{
 
71
        if (bucket)
 
72
                bucket->release();
 
73
                
 
74
        if (s3Prot)
 
75
                s3Prot->release();
 
76
}
 
77
        
 
78
//-------------------------------
 
79
const char *MSCloudInfo::getServer() 
 
80
 
81
        return s3Prot->s3_getServer();
 
82
}
 
83
 
 
84
//-------------------------------
 
85
const char *MSCloudInfo::getBucket() 
 
86
 
87
        return bucket->getCString();
 
88
}
 
89
 
 
90
//-------------------------------
 
91
const char *MSCloudInfo::getPublicKey() 
 
92
 
93
        return s3Prot->s3_getPublicKey();
 
94
}
 
95
 
 
96
//-------------------------------
 
97
const char *MSCloudInfo::getPrivateKey() 
 
98
 
99
        return s3Prot->s3_getPrivateKey();
 
100
}
 
101
 
 
102
//-------------------------------
 
103
CSString *MSCloudInfo::getSignature(const char *key, const char *content_type, uint32_t *s3AuthorizationTime)
 
104
{
 
105
        return s3Prot->s3_getAuthorization(bucket->getCString(), key, content_type, s3AuthorizationTime);
 
106
}
 
107
 
 
108
//-------------------------------
 
109
CSString *MSCloudInfo::getDataURL(const char *key, int keep_alive)
 
110
{
 
111
        return s3Prot->s3_getDataURL(bucket->getCString(), key, keep_alive);
 
112
}
 
113
 
 
114
//-------------------------------
 
115
void MSCloudInfo::send(CSInputStream *input, const char *key, off_t size)
 
116
{
 
117
        CSVector *headers;
 
118
        headers = s3Prot->s3_send(input, bucket->getCString(), key, size);
 
119
        headers->release();
 
120
}
 
121
 
 
122
//-------------------------------
 
123
CSVector *MSCloudInfo::list(const char *key_prefix, uint32_t max)
 
124
{
 
125
        return s3Prot->s3_list(bucket->getCString(), key_prefix, max);
 
126
}
 
127
 
 
128
//-------------------------------
 
129
void MSCloudInfo::receive(CSOutputStream *output, const char *key)
 
130
{
 
131
        bool found;
 
132
        CSVector *headers;
 
133
        
 
134
        headers = s3Prot->s3_receive(output, bucket->getCString(), key, &found);
 
135
        headers->release();
 
136
        if (!found) {
 
137
                CSStringBuffer *err;
 
138
                enter_();
 
139
                
 
140
                new_(err, CSStringBuffer());
 
141
                push_(err);
 
142
                err->append("S3 object not found: ");
 
143
                err->append(getServer());
 
144
                err->append("/");
 
145
                err->append(bucket->getCString());
 
146
                err->append("/");
 
147
                err->append(key);
 
148
 
 
149
                CSException::throwException(CS_CONTEXT, CS_ERR_GENERIC_ERROR, err->getCString());
 
150
                release_(err);
 
151
                outer_();
 
152
        }
 
153
}
 
154
 
 
155
//-------------------------------
 
156
void MSCloudInfo::cDelete(const char *key)
 
157
{
 
158
        s3Prot->s3_delete(bucket->getCString(), key);
 
159
}
 
160
 
 
161
//-------------------------------
 
162
void MSCloudInfo::copy(MSCloudInfo *dst_cloud, const char *dst_key, const char *src_key)
 
163
{
 
164
        enter_();
 
165
        push_(dst_cloud);
 
166
        
 
167
        s3Prot->s3_copy(dst_cloud->getServer() ,dst_cloud->bucket->getCString(), dst_key, bucket->getCString(), src_key);
 
168
 
 
169
        release_(dst_cloud);
 
170
        exit_();
 
171
}
 
172
 
 
173
//==============================
 
174
CloudDB::CloudDB(uint32_t db_id):
 
175
        isBackup(false),
 
176
        blob_recovery_no(0),
 
177
        blob_db_id(db_id),
 
178
        dfltCloudRefId(0),
 
179
        backupCloud(NULL),
 
180
        backupInfo(NULL),
 
181
        clObjectKey(NULL)
 
182
{
 
183
        enter_();
 
184
 
 
185
        keep_alive = (5 * 60); // default URL keep alive in seconds.
 
186
        new_(clObjectKey, CSStringBuffer());
 
187
        clObjectKey->setLength(base_key_size);
 
188
                
 
189
        exit_();
 
190
}
 
191
 
 
192
//-------------------------------
 
193
CloudDB::~CloudDB()
 
194
{
 
195
        
 
196
        if (backupInfo)
 
197
                backupInfo->release();
 
198
                
 
199
        if (backupCloud)
 
200
                backupCloud->release();
 
201
                
 
202
        if (clObjectKey)
 
203
                clObjectKey->release();
 
204
                
 
205
}
 
206
//-------------------------------
 
207
MSBackupInfo *CloudDB::cl_getBackupInfo()
 
208
 
209
        if (backupInfo)
 
210
                backupInfo->retain();
 
211
                
 
212
        return backupInfo;
 
213
}
 
214
 
 
215
//-------------------------------
 
216
void CloudDB::cl_clearBackupInfo(){ backupInfo->release(); backupInfo = NULL;}
 
217
 
 
218
//-------------------------------
 
219
void CloudDB::cl_createDB()
 
220
{
 
221
// This is a no-op. 
 
222
}
 
223
 
 
224
//-------------------------------
 
225
// Restore all the 
 
226
void CloudDB::cl_restoreDB()
 
227
{
 
228
        CSVector *list;
 
229
        CSString *key;
 
230
        CloudObjectKey *src_objectKey, *dst_objectKey;
 
231
        CloudKeyRec             cloudKey;
 
232
        uint32_t                        src_cloudRef, dst_cloudRef = 0;
 
233
        MSBackupInfo    *backup_info;
 
234
        MSCloudInfo             *src_cloud, *dst_cloud;
 
235
        enter_();
 
236
 
 
237
        if (!blob_recovery_no)
 
238
                exit_(); // nothing to do.
 
239
 
 
240
        backup_info = MSBackupInfo::getBackupInfo(blob_recovery_no);
 
241
        push_(backup_info);
 
242
        
 
243
        src_cloudRef = backup_info->getcloudRef();
 
244
        src_cloud = MSCloudInfo::getCloudInfo(src_cloudRef);
 
245
        push_(src_cloud);
 
246
        
 
247
        new_(dst_objectKey, CloudObjectKey(blob_db_id));
 
248
        push_(dst_objectKey);
 
249
        
 
250
        // Get the key for the backup BLOB
 
251
        new_(src_objectKey, CloudObjectKey(blob_db_id));
 
252
        push_(src_objectKey);
 
253
        src_objectKey->setObjectKey(NULL, backup_info->getcloudBackupNo(), backup_info->getDatabaseId()); 
 
254
 
 
255
        // Get a list of all the BLOBs that were backed up.
 
256
        list = src_cloud->list(src_objectKey->getCString());
 
257
        release_(src_objectKey);
 
258
        push_(list);
 
259
        
 
260
        // Go through the list copying the keys.
 
261
        try_(a) {
 
262
                dst_cloudRef = src_cloudRef;
 
263
                dst_cloud = src_cloud;
 
264
                dst_cloud->retain();
 
265
        
 
266
                while ((key = (CSString*)(list->take(0))) ) {
 
267
                        push_(key);
 
268
 
 
269
                        // The source key name must be parsed to get its
 
270
                        // destination cloud reference. The destination for
 
271
                        // the BLOBs may not all be in the same cloud. 
 
272
                        CloudObjectKey::parseObjectKey(key->getCString(), &cloudKey);
 
273
                        
 
274
                        // Reset the destination cloud if required.
 
275
                        if (cloudKey.cloud_ref != dst_cloudRef) {
 
276
                                if (dst_cloud) {
 
277
                                        dst_cloud->release();
 
278
                                        dst_cloud = NULL;
 
279
                                }
 
280
                                dst_cloudRef =  cloudKey.cloud_ref;
 
281
                                dst_cloud = MSCloudInfo::getCloudInfo(dst_cloudRef);
 
282
                        }
 
283
 
 
284
                        // Copy the BLOB to the recovered database.
 
285
                        dst_objectKey->setObjectKey(&cloudKey);
 
286
                        src_cloud->copy(RETAIN(dst_cloud), dst_objectKey->getCString(), key->getCString());
 
287
                        release_(key);
 
288
                        
 
289
                }
 
290
        }
 
291
        
 
292
        finally_(a) {
 
293
                if (dst_cloud)
 
294
                        dst_cloud->release();
 
295
        }
 
296
        cont_(a);
 
297
        
 
298
        blob_recovery_no = 0;
 
299
        release_(list);
 
300
        release_(dst_objectKey);        
 
301
        release_(src_cloud);
 
302
        release_(backup_info);
 
303
        exit_();        
 
304
}
 
305
 
 
306
//-------------------------------
 
307
uint32_t CloudDB::cl_getNextBackupNumber(uint32_t cloud_ref)
 
308
{
 
309
        CloudObjectKey *objectKey;
 
310
        CSVector *list;
 
311
        uint32_t backup_no = 0, size = 1;
 
312
        MSCloudInfo *s3Cloud;
 
313
        enter_();
 
314
 
 
315
        s3Cloud = MSCloudInfo::getCloudInfo((cloud_ref)?cloud_ref:dfltCloudRefId);
 
316
        push_(s3Cloud);
 
317
        
 
318
        new_(objectKey, CloudObjectKey(blob_db_id));
 
319
        push_(objectKey);
 
320
 
 
321
        // Find the next available backup number
 
322
        while (size) {
 
323
                backup_no++;
 
324
                objectKey->setObjectKey(NULL, backup_no); // use the key prefix with the backup number for listing.
 
325
                list = s3Cloud->list(objectKey->getCString(), 1);
 
326
                size = list->size();
 
327
                list->release();
 
328
        }
 
329
        
 
330
        release_(objectKey);
 
331
        release_(s3Cloud);
 
332
 
 
333
        return_(backup_no);
 
334
}
 
335
 
 
336
//-------------------------------
 
337
void CloudDB::cl_backupBLOB(CloudKeyPtr key)
 
338
{
 
339
        CloudObjectKey *src_objectKey, *dst_objectKey;
 
340
        uint32_t cloudRef, backupNo;
 
341
        MSCloudInfo *src_cloud, *dst_cloud;
 
342
        enter_();
 
343
 
 
344
        ASSERT(backupInfo);
 
345
        
 
346
        if ((cloudRef = backupInfo->getcloudRef()) == 0) {
 
347
                backupInfo->setcloudRef(dfltCloudRefId);
 
348
                cloudRef = dfltCloudRefId;
 
349
        }
 
350
                
 
351
        if ((backupNo = backupInfo->getcloudBackupNo()) == 0) {         
 
352
                backupNo = cl_getNextBackupNumber(cloudRef);
 
353
                backupInfo->setcloudBackupNo(backupNo);
 
354
        }
 
355
        
 
356
        // Set the source object's key
 
357
        new_(src_objectKey, CloudObjectKey(blob_db_id));
 
358
        push_(src_objectKey);
 
359
        src_objectKey->setObjectKey(key);
 
360
 
 
361
        // Set the destination object's key
 
362
        new_(dst_objectKey, CloudObjectKey(blob_db_id));
 
363
        push_(dst_objectKey);
 
364
        dst_objectKey->setObjectKey(key, backupNo);
 
365
 
 
366
        // Get the source cloud
 
367
        src_cloud = MSCloudInfo::getCloudInfo((key->cloud_ref)?key->cloud_ref:dfltCloudRefId);
 
368
        push_(src_cloud);
 
369
        
 
370
        // Copy the object to the destination cloud
 
371
        dst_cloud = MSCloudInfo::getCloudInfo(cloudRef);
 
372
        src_cloud->copy(dst_cloud, dst_objectKey->getCString(), src_objectKey->getCString());
 
373
 
 
374
        release_(src_cloud);
 
375
        release_(dst_objectKey);
 
376
        release_(src_objectKey);
 
377
        exit_();
 
378
}
 
379
 
 
380
//-------------------------------
 
381
void CloudDB::cl_restoreBLOB(CloudKeyPtr key, uint32_t backup_db_id)
 
382
{
 
383
        CloudObjectKey *src_objectKey, *dst_objectKey;
 
384
        uint32_t cloudRef, backupNo;
 
385
        MSCloudInfo *src_cloud, *dst_cloud;
 
386
        enter_();
 
387
 
 
388
        ASSERT(backupInfo);
 
389
        
 
390
        if ((cloudRef = backupInfo->getcloudRef()) == 0) {
 
391
                backupInfo->setcloudRef(dfltCloudRefId);
 
392
                cloudRef = dfltCloudRefId;
 
393
        }
 
394
                
 
395
        if ((backupNo = backupInfo->getcloudBackupNo()) == 0) {         
 
396
                backupNo = cl_getNextBackupNumber(cloudRef);
 
397
                backupInfo->setcloudBackupNo(backupNo);
 
398
        }
 
399
        
 
400
        // Set the source object's key
 
401
        new_(src_objectKey, CloudObjectKey(backup_db_id));
 
402
        push_(src_objectKey);
 
403
        src_objectKey->setObjectKey(key, backupNo);
 
404
 
 
405
        // Set the destination object's key
 
406
        new_(dst_objectKey, CloudObjectKey(blob_db_id));
 
407
        push_(dst_objectKey);
 
408
        dst_objectKey->setObjectKey(key);
 
409
 
 
410
        // Get the source cloud
 
411
        src_cloud = MSCloudInfo::getCloudInfo(cloudRef);
 
412
        push_(src_cloud);
 
413
        
 
414
        // Copy the object to the destination cloud
 
415
        dst_cloud = MSCloudInfo::getCloudInfo((key->cloud_ref)?key->cloud_ref:dfltCloudRefId);
 
416
        src_cloud->copy(dst_cloud, dst_objectKey->getCString(), src_objectKey->getCString());
 
417
 
 
418
        release_(src_cloud);
 
419
        release_(dst_objectKey);
 
420
        release_(src_objectKey);
 
421
        exit_();
 
422
}
 
423
 
 
424
//-------------------------------
 
425
// Drop database deletes all objects with the database key prefix
 
426
void CloudDB::cl_dropDB()
 
427
{
 
428
        CSVector *list;
 
429
        CSString *key;
 
430
        CloudObjectKey *objectKey;      
 
431
        MSCloudInfo *s3Cloud = NULL;
 
432
        int i;
 
433
        const char *key_str;
 
434
        
 
435
        enter_();
 
436
        new_(objectKey, CloudObjectKey(blob_db_id));
 
437
        push_(objectKey);
 
438
        
 
439
        lock_(MSCloudInfo::gCloudInfo);
 
440
 
 
441
        if (isBackup) {
 
442
                uint32_t backup_no;
 
443
                if (backupInfo && (backup_no = backupInfo->getcloudBackupNo())) {
 
444
                        objectKey->setObjectKey(NULL, backup_no); // use the key prefix for the backup for listing.
 
445
                        if ((s3Cloud = MSCloudInfo::getCloudInfo(backupInfo->getcloudRef())))
 
446
                                push_(s3Cloud);
 
447
                }
 
448
        } else {
 
449
                objectKey->setObjectKey(); // use the key prefix for listing.
 
450
                i = 0;
 
451
                s3Cloud = (MSCloudInfo*)MSCloudInfo::gCloudInfo->itemAt(i++); // <-- unreferenced object 
 
452
        }
 
453
                
 
454
        key_str = objectKey->getCString();
 
455
 
 
456
        // For non backup BLOBs all known clouds must be searched 
 
457
        // for possible BLOBs and deleted. The BLOBs belonging to a backup
 
458
        // will ever only be in one cloud storage location.
 
459
        while (s3Cloud) {
 
460
                list = s3Cloud->list(key_str);
 
461
                push_(list);
 
462
                
 
463
                // Go through the list deleting the keys.
 
464
                while ((key = (CSString*)(list->take(0))) ) {
 
465
                        push_(key);
 
466
                        s3Cloud->cDelete(key->getCString());
 
467
                        release_(key);
 
468
                }
 
469
                
 
470
                release_(list);
 
471
                if (isBackup) {
 
472
                        release_(s3Cloud); // Only the backup s3Cloud needs to be released.
 
473
                        s3Cloud = NULL;
 
474
                } else
 
475
                        s3Cloud = (MSCloudInfo*)MSCloudInfo::gCloudInfo->itemAt(i++);// <-- unreferenced object
 
476
        }
 
477
        
 
478
done:
 
479
        unlock_(MSCloudInfo::gCloudInfo);
 
480
        release_(objectKey);
 
481
        exit_();
 
482
}
 
483
 
 
484
//-------------------------------
 
485
void CloudDB::cl_putData(CloudKeyPtr key, CSInputStream *stream, off_t size)
 
486
{
 
487
        CloudObjectKey *objectKey;
 
488
        MSCloudInfo *s3Cloud;
 
489
        
 
490
        enter_();
 
491
        
 
492
        push_(stream);
 
493
        
 
494
        new_(objectKey, CloudObjectKey(blob_db_id));
 
495
        push_(objectKey);
 
496
        
 
497
        objectKey->setObjectKey(key);
 
498
        
 
499
        s3Cloud = MSCloudInfo::getCloudInfo((key->cloud_ref)?key->cloud_ref:dfltCloudRefId);
 
500
        push_(s3Cloud);
 
501
        s3Cloud->send(RETAIN(stream), objectKey->getCString(), size);
 
502
        release_(s3Cloud);
 
503
        
 
504
        release_(objectKey);
 
505
        release_(stream);
 
506
        
 
507
        exit_();
 
508
}
 
509
 
 
510
//-------------------------------
 
511
off_t CloudDB::cl_getData(CloudKeyPtr key,  char *buffer, off_t size)
 
512
{       
 
513
        CloudObjectKey *objectKey;
 
514
        CSStaticMemoryOutputStream *output;
 
515
        MSCloudInfo *s3Cloud;
 
516
        enter_();
 
517
        
 
518
        new_(objectKey, CloudObjectKey(blob_db_id));
 
519
        push_(objectKey);
 
520
        
 
521
        s3Cloud = MSCloudInfo::getCloudInfo(key->cloud_ref);
 
522
        push_(s3Cloud);
 
523
 
 
524
        new_(output, CSStaticMemoryOutputStream((u_char *)buffer, size));
 
525
        push_(output);
 
526
        
 
527
        objectKey->setObjectKey(key);
 
528
        
 
529
        s3Cloud->receive(RETAIN(output), objectKey->getCString());      
 
530
        size = output->getSize();
 
531
        release_(output);
 
532
        
 
533
        release_(s3Cloud);
 
534
        release_(objectKey);
 
535
        return_(size);
 
536
}
 
537
 
 
538
//-------------------------------
 
539
void CloudDB::cl_deleteData(CloudKeyPtr key)
 
540
{
 
541
        MSCloudInfo *s3Cloud;
 
542
        CloudObjectKey *objectKey;
 
543
        enter_();
 
544
        
 
545
        new_(objectKey, CloudObjectKey(blob_db_id));
 
546
        push_(objectKey);
 
547
        
 
548
        s3Cloud = MSCloudInfo::getCloudInfo(key->cloud_ref);
 
549
        push_(s3Cloud);
 
550
 
 
551
        objectKey->setObjectKey(key);
 
552
 
 
553
        s3Cloud->cDelete(objectKey->getCString());      
 
554
        
 
555
        release_(s3Cloud);
 
556
        release_(objectKey);
 
557
 
 
558
        exit_();
 
559
}
 
560
 
 
561
//-------------------------------
 
562
CSString *CloudDB::cl_getDataURL(CloudKeyPtr key)
 
563
{
 
564
        CloudObjectKey *objectKey;
 
565
        CSString *url;
 
566
        MSCloudInfo *s3Cloud;
 
567
        enter_();
 
568
        
 
569
        new_(objectKey, CloudObjectKey(blob_db_id));
 
570
        push_(objectKey);
 
571
        
 
572
        objectKey->setObjectKey(key);
 
573
        
 
574
        s3Cloud = MSCloudInfo::getCloudInfo(key->cloud_ref);  
 
575
        push_(s3Cloud);
 
576
                
 
577
        url = s3Cloud->getDataURL(objectKey->getCString(), keep_alive);
 
578
        
 
579
        release_(s3Cloud);
 
580
        release_(objectKey);
 
581
 
 
582
        return_(url);
 
583
}
 
584
 
 
585
//-------------------------------
 
586
CSString *CloudDB::cl_getSignature(CloudKeyPtr key, CSString *content_type_arg, uint32_t *s3AuthorizationTime)
 
587
{
 
588
        CSString *signature;
 
589
        CloudObjectKey *objectKey;
 
590
        const char *content_type = NULL;
 
591
        MSCloudInfo *s3Cloud;
 
592
        enter_();
 
593
        
 
594
        new_(objectKey, CloudObjectKey(blob_db_id));
 
595
        push_(objectKey);
 
596
        
 
597
        if (content_type_arg) {
 
598
                push_(content_type_arg);
 
599
                content_type = content_type_arg->getCString();
 
600
        }
 
601
        
 
602
        objectKey->setObjectKey(key);
 
603
        s3Cloud = MSCloudInfo::getCloudInfo(key->cloud_ref);  
 
604
        push_(s3Cloud);
 
605
        
 
606
        signature = s3Cloud->getSignature(objectKey->getCString(), content_type, s3AuthorizationTime);
 
607
        
 
608
        if (content_type_arg) 
 
609
                release_(content_type_arg);
 
610
 
 
611
        release_(s3Cloud);
 
612
        release_(objectKey);
 
613
        
 
614
        return_(signature);
 
615
}
 
616
 
 
617
//==============================
 
618