« back to all changes in this revision
Viewing changes to plugin/pbms/src/cslib/CSS3Protocol.cc
- browse files at revision 1643.1.1
- compare with another revision
- compare with revision 1548.2.18
- download diff
- download tarball
- view history from revision 1643.1.1
- Committer: Monty Taylor
- Date: 2010-07-04 20:02:43 UTC
- mfrom: (1548.2.40 drizzle_pbms)
- mto: This revision was merged to the branch mainline in revision 1644.
- Revision ID: mordred@inaugust.com-20100704200243-2vkq9gi6ysauj2tb
Merge PBMS from Barry.
- files added:
- plugin/pbms
- plugin/pbms/src
- plugin/pbms/src/cslib
- files modified:
- config/make-lint.py
added
removed
plugin/pbms/src/cslib/CSS3Protocol.cc
1
/* Copyright (c) 2009 PrimeBase Technologies GmbH, Germany
2
*
3
* PrimeBase Media Stream for MySQL
4
*
5
* This program is free software; you can redistribute it and/or modify
6
* it under the terms of the GNU General Public License as published by
7
* the Free Software Foundation; either version 2 of the License, or
8
* (at your option) any later version.
9
*
10
* This program is distributed in the hope that it will be useful,
11
* but WITHOUT ANY WARRANTY; without even the implied warranty of
12
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13
* GNU General Public License for more details.
14
*
15
* You should have received a copy of the GNU General Public License
16
* along with this program; if not, write to the Free Software
17
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
18
*
19
* Created by Barry Leslie on 10/02/09.
20
*
21
*/
22
#include "CSConfig.h"
23
#include <inttypes.h>
24
#include <stdlib.h>
25
26
#include <curl/curl.h>
27
28
#include "CSGlobal.h"
29
#include "CSString.h"
30
#include "CSStrUtil.h"
31
#include "CSEncode.h"
32
#include "CSS3Protocol.h"
33
#include "CSXML.h"
34
35
#ifdef S3_UNIT_TEST
36
//#define SHOW_SIGNING
37
// Uncomment this line to trace network action during request. Very Usefull!!
38
//#define DEBUG_CURL
39
#define DUMP_ERRORS
40
#endif
41
42
//#define DEBUG_CURL
43
#define DUMP_ERRORS
44
//#define SHOW_SIGNING
45
46
#define HEX_CHECKSUM_VALUE_SIZE (2 *CHECKSUM_VALUE_SIZE)
47
48
#define THROW_CURL_IF(v) { if (v) CSException::throwException(CS_CONTEXT, CS_ERR_GENERIC_ERROR, ms_curl_error);}
49
50
//-------------------------------
51
static const char *retryCodes[] = {
52
"ExpiredToken",
53
"InternalError",
54
"OperationAborted",
55
"RequestTimeout",
56
"SlowDown",
57
NULL
58
};
59
60
//======================================
61
static size_t receive_data(void *ptr, size_t size, size_t nmemb, void *stream);
62
static size_t receive_header(void *ptr, size_t size, size_t nmemb, void *stream);
63
static size_t send_callback(void *ptr, size_t size, size_t nmemb, void *stream);
64
65
class S3ProtocolCon : CSXMLBuffer, public CSObject {
66
67
private:
68
69
virtual bool openNode(char *path, char *value) {
70
if (value && *value && (strcmp(path,"/error/code/") == 0)) {
71
printf("S3 ERROR Code: %s\n", value);
72
for (int i = 0; retryCodes[i] && !ms_retry; i++)
73
ms_retry = (strcmp(value, retryCodes[i]) == 0);
74
75
if (ms_retry && !strcmp("slowdown", value))
76
ms_slowDown = true;
77
} else if (value && *value && (strcmp(path,"/error/message/") == 0)) {
78
printf("S3 ERROR MESSAGE: %s\n", value);
79
}
80
return true;
81
}
82
83
virtual bool closeNode(char *path) {
84
(void)path;
85
return true;
86
}
87
88
virtual bool addAttribute(char *path, char *name, char *value) {
89
(void)path;
90
(void)name;
91
(void)value;
92
return true;
93
}
94
95
//-------------------------------
96
void parse_s3_error()
97
{
98
enter_();
99
100
if (!ms_errorReply)
101
CSException::throwException(CS_CONTEXT, CS_ERR_GENERIC_ERROR, "Missing HTTP reply: possible S3 connection failure.");
102
103
#ifdef DUMP_ERRORS
104
printf("ms_errorReply:\n===========\n%s\n===========\n", ms_errorReply->getCString());
105
#endif
106
107
if (!parseData(ms_errorReply->getCString(), ms_errorReply->length(), 0)){
108
int err;
109
char *msg;
110
111
getError(&err, &msg);
112
CSException::throwException(CS_CONTEXT, CS_ERR_GENERIC_ERROR, msg);
113
}
114
115
exit_();
116
}
117
118
public:
119
120
CSHTTPHeaders ms_reply_headers;
121
CSStringBuffer ms_buffer; // A scratch buffer
122
123
CURL *ms_curl;
124
struct curl_slist *ms_header_list; // A curl list of headers to be sent with the next request.
125
126
CSInputStream *ms_inputStream;
127
CSOutputStream *ms_outputStream;
128
129
CSMd5 ms_md5;
130
char ms_s3Checksum[HEX_CHECKSUM_VALUE_SIZE +1];
131
132
bool ms_notFound; // True if the object could not be found
133
bool ms_retry; // True if the request failed with a retry error.
134
bool ms_slowDown;
135
136
CSStringBuffer *ms_errorReply;
137
char ms_curl_error[CURL_ERROR_SIZE];
138
139
off64_t ms_data_size;
140
141
unsigned int ms_replyStatus;
142
bool ms_throw_error; // Gets set if an exception occurs in a callback.
143
bool ms_old_libcurl;
144
char *ms_safe_url;
145
146
S3ProtocolCon():
147
ms_curl(NULL),
148
ms_header_list(NULL),
149
ms_inputStream(NULL),
150
ms_outputStream(NULL),
151
ms_notFound(false),
152
ms_retry(false),
153
ms_slowDown(false),
154
ms_errorReply(NULL),
155
ms_data_size(0),
156
ms_replyStatus(0),
157
ms_throw_error(false),
158
ms_old_libcurl(false),
159
ms_safe_url(NULL)
160
{
161
162
ms_curl = curl_easy_init();
163
if (!ms_curl)
164
CSException::throwException(CS_CONTEXT, CS_ERR_GENERIC_ERROR, "curl_easy_init() failed.");
165
166
curl_version_info_data *curl_ver = curl_version_info(CURLVERSION_NOW);
167
168
// libCurl versions prior to 7.17.0 did not make copies of strings passed into curl_easy_setopt()
169
// If this version requirement is a problem I can do this myself, if I have to, I guess. :(
170
if (curl_ver->version_num < 0X071700 ) {
171
ms_old_libcurl = true;
172
173
//char msg[200];
174
//snprintf(msg, 200, "libcurl version %s is too old, require version 7.17.0 or newer.", curl_ver->version);
175
//CSException::throwException(CS_CONTEXT, CS_ERR_GENERIC_ERROR, msg);
176
}
177
178
if (curl_easy_setopt(ms_curl, CURLOPT_ERRORBUFFER, ms_curl_error))
179
CSException::throwException(CS_CONTEXT, CS_ERR_GENERIC_ERROR, "curl_easy_setopt(CURLOPT_ERRORBUFFER) failed.");
180
181
#ifdef DEBUG_CURL
182
curl_easy_setopt(ms_curl, CURLOPT_VERBOSE, 1L);
183
#endif
184
//THROW_CURL_IF(curl_easy_setopt(ms_curl, CURLOPT_TCP_NODELAY, 1L));
185
186
187
THROW_CURL_IF(curl_easy_setopt(ms_curl, CURLOPT_NOPROGRESS, 1L));
188
THROW_CURL_IF(curl_easy_setopt(ms_curl, CURLOPT_WRITEFUNCTION, receive_data));
189
THROW_CURL_IF(curl_easy_setopt(ms_curl, CURLOPT_READFUNCTION, send_callback));
190
THROW_CURL_IF(curl_easy_setopt(ms_curl, CURLOPT_HEADERFUNCTION, receive_header));
191
THROW_CURL_IF(curl_easy_setopt(ms_curl, CURLOPT_WRITEDATA, this));
192
THROW_CURL_IF(curl_easy_setopt(ms_curl, CURLOPT_READDATA, this));
193
THROW_CURL_IF(curl_easy_setopt(ms_curl, CURLOPT_WRITEHEADER, this));
194
195
196
THROW_CURL_IF(curl_easy_setopt(ms_curl, CURLOPT_FOLLOWLOCATION, 1L)); // Follow redirects.
197
198
}
199
200
~S3ProtocolCon()
201
{
202
if (ms_curl)
203
curl_easy_cleanup(ms_curl);
204
if (ms_header_list)
205
curl_slist_free_all(ms_header_list);
206
if (ms_inputStream)
207
ms_inputStream->release();
208
if (ms_outputStream)
209
ms_outputStream->release();
210
if (ms_errorReply)
211
ms_errorReply->release();
212
213
ms_reply_headers.clearHeaders();
214
215
if (ms_safe_url)
216
cs_free(ms_safe_url);
217
}
218
219
inline void check_reply_status()
220
{
221
if (ms_replyStatus > 199 && ms_replyStatus < 300)
222
return;
223
224
225
226
switch (ms_replyStatus) {
227
case 200:
228
case 204: // No Content
229
//case 301: // Moved Permanently
230
//case 307: // Temporary Redirect
231
break;
232
case 404: // Not Found
233
ms_notFound = true;
234
break;
235
case 500:
236
ms_retry = true;
237
break;
238
default: {
239
parse_s3_error();
240
241
242
243
if (!ms_retry) {
244
enter_();
245
CSException::throwException(CS_CONTEXT, CS_ERR_GENERIC_ERROR, ms_errorReply->getCString());
246
outer_();
247
} else if (ms_slowDown) {
248
enter_();
249
CSException::logException(CS_CONTEXT, CS_ERR_GENERIC_ERROR, "S3 slow down request.");
250
self->sleep(10); // sleep for 1/100 second.
251
outer_();
252
}
253
}
254
}
255
256
}
257
258
259
inline void ms_reset()
260
{
261
// Remove any old headers
262
if (ms_header_list) {
263
curl_slist_free_all(ms_header_list);
264
ms_header_list = NULL;
265
}
266
267
ms_reply_headers.clearHeaders();
268
ms_replyStatus = 0;
269
ms_throw_error = false;
270
if (ms_errorReply)
271
ms_errorReply->setLength(0);
272
273
ms_s3Checksum[0] = 0;
274
ms_notFound = false;
275
ms_retry = false;
276
277
if (ms_outputStream) {
278
ms_outputStream->release();
279
ms_outputStream = NULL;
280
}
281
if (ms_inputStream) {
282
ms_inputStream->release();
283
ms_inputStream = NULL;
284
}
285
286
if (ms_safe_url) {
287
cs_free(ms_safe_url);
288
ms_safe_url = NULL;
289
}
290
}
291
292
inline void ms_setHeader(const char *header)
293
{
294
ms_header_list = curl_slist_append(ms_header_list, header);
295
if (!ms_header_list)
296
CSException::throwException(CS_CONTEXT, CS_ERR_GENERIC_ERROR, "curl_slist_append() failed.");
297
}
298
299
300
private:
301
inline const char *safe_url(const char *url)
302
{
303
if (ms_old_libcurl == false)
304
return url;
305
306
if (ms_safe_url) {
307
cs_free(ms_safe_url);
308
ms_safe_url = NULL;
309
}
310
ms_safe_url = cs_strdup(url);
311
return ms_safe_url;
312
}
313
314
public:
315
inline void ms_setURL(const char *url)
316
{
317
//printf("URL: \"%s\n", url);
318
THROW_CURL_IF(curl_easy_setopt(ms_curl, CURLOPT_URL, safe_url(url)));
319
}
320
321
inline void ms_execute_delete_request()
322
{
323
CURLcode rtc;
324
325
THROW_CURL_IF(curl_easy_setopt(ms_curl, CURLOPT_HTTPHEADER, ms_header_list));
326
THROW_CURL_IF(curl_easy_setopt(ms_curl, CURLOPT_CUSTOMREQUEST, "DELETE"));
327
328
rtc = curl_easy_perform(ms_curl);
329
330
THROW_CURL_IF(curl_easy_setopt(ms_curl, CURLOPT_CUSTOMREQUEST, NULL)); // IMPORTANT: Reset this to it's default value
331
332
if (rtc && !ms_throw_error)
333
CSException::throwException(CS_CONTEXT, CS_ERR_GENERIC_ERROR, ms_curl_error);
334
335
if (ms_throw_error) {
336
enter_();
337
throw_();
338
outer_();
339
}
340
341
check_reply_status();
342
}
343
344
inline void ms_execute_copy_request()
345
{
346
CURLcode rtc;
347
348
THROW_CURL_IF(curl_easy_setopt(ms_curl, CURLOPT_HTTPHEADER, ms_header_list));
349
THROW_CURL_IF(curl_easy_setopt(ms_curl, CURLOPT_INFILESIZE_LARGE, 0));
350
THROW_CURL_IF(curl_easy_setopt(ms_curl, CURLOPT_UPLOAD, 1L));
351
352
rtc = curl_easy_perform(ms_curl);
353
354
if (rtc && !ms_throw_error)
355
CSException::throwException(CS_CONTEXT, CS_ERR_GENERIC_ERROR, ms_curl_error);
356
357
if (ms_throw_error) {
358
enter_();
359
throw_();
360
outer_();
361
}
362
363
check_reply_status();
364
}
365
366
inline void ms_execute_get_request(CSOutputStream *output)
367
{
368
enter_();
369
370
push_(output);
371
THROW_CURL_IF(curl_easy_setopt(ms_curl, CURLOPT_HTTPHEADER, ms_header_list));
372
THROW_CURL_IF(curl_easy_setopt(ms_curl, CURLOPT_HTTPGET, 1L));
373
374
ms_outputStream = output;
375
if (curl_easy_perform(ms_curl) && !ms_throw_error) {
376
ms_outputStream = NULL;
377
CSException::throwException(CS_CONTEXT, CS_ERR_GENERIC_ERROR, ms_curl_error);
378
}
379
ms_outputStream = NULL;
380
release_(output);
381
382
if (ms_throw_error)
383
throw_();
384
385
check_reply_status();
386
exit_();
387
}
388
389
inline void ms_execute_put_request(CSInputStream *input, off64_t size, Md5Digest *digest)
390
{
391
enter_();
392
393
push_(input);
394
THROW_CURL_IF(curl_easy_setopt(ms_curl, CURLOPT_HTTPHEADER, ms_header_list));
395
THROW_CURL_IF(curl_easy_setopt(ms_curl, CURLOPT_INFILESIZE_LARGE, size));
396
THROW_CURL_IF(curl_easy_setopt(ms_curl, CURLOPT_UPLOAD, 1L));
397
//THROW_CURL_IF(curl_easy_setopt(ms_curl, CURLOPT_POSTFIELDSIZE_LARGE, size));
398
//THROW_CURL_IF(curl_easy_setopt(ms_curl, CURLOPT_POST, 1L));
399
400
ms_md5.md5_init();
401
402
ms_data_size = size;
403
ms_inputStream = input;
404
if (curl_easy_perform(ms_curl) && !ms_throw_error) {
405
ms_inputStream = NULL;
406
CSException::throwException(CS_CONTEXT, CS_ERR_GENERIC_ERROR, ms_curl_error);
407
}
408
ms_inputStream = NULL;
409
release_(input);
410
411
412
if (ms_throw_error)
413
throw_();
414
415
check_reply_status();
416
417
// Check that the checksums agree
418
char checksum[HEX_CHECKSUM_VALUE_SIZE +1];
419
420
ms_md5.md5_digest(digest);
421
cs_bin_to_hex(HEX_CHECKSUM_VALUE_SIZE, checksum, CHECKSUM_VALUE_SIZE, digest->val);
422
checksum[HEX_CHECKSUM_VALUE_SIZE] = 0;
423
424
cs_strToUpper(ms_s3Checksum);
425
if (strcmp(checksum, ms_s3Checksum)) {
426
// The request should be restarted in this case.
427
ms_retry = true;
428
CSException::logException(CS_CONTEXT, CS_ERR_CHECKSUM_ERROR, "Calculated checksum did not match S3 checksum");
429
}
430
431
exit_();
432
}
433
434
};
435
436
//======================================
437
438
439
440
441
//======================================
442
CSS3Protocol::CSS3Protocol():
443
s3_server(NULL),
444
s3_public_key(NULL),
445
s3_private_key(NULL),
446
s3_maxRetrys(5)
447
{
448
new_(s3_server, CSStringBuffer());
449
s3_server->append("s3.amazonaws.com/");
450
451
s3_public_key = CSString::newString("");
452
s3_private_key = CSString::newString("");
453
454
}
455
456
//------------------
457
CSS3Protocol::~CSS3Protocol()
458
{
459
if (s3_server)
460
s3_server->release();
461
462
if (s3_public_key)
463
s3_public_key->release();
464
465
if (s3_private_key)
466
s3_private_key->release();
467
}
468
469
//------------------
470
CSString *CSS3Protocol::s3_getSignature(const char *verb,
471
const char *md5,
472
const char *content_type,
473
const char *date,
474
const char *bucket,
475
const char *key,
476
CSString *headers
477
)
478
{
479
CSStringBuffer *s3_buffer;
480
enter_();
481
if (headers)
482
push_(headers);
483
484
new_(s3_buffer, CSStringBuffer());
485
push_(s3_buffer);
486
487
s3_buffer->setLength(0);
488
s3_buffer->append(verb);
489
s3_buffer->append("\n");
490
if (md5) s3_buffer->append(md5);
491
s3_buffer->append("\n");
492
if (content_type) s3_buffer->append(content_type);
493
s3_buffer->append("\n");
494
s3_buffer->append(date);
495
if (headers) {
496
// Note: headers are assumed to be in lower case, sorted, and containing no white space.
497
s3_buffer->append("\n");
498
s3_buffer->append(headers->getCString());
499
}
500
s3_buffer->append("\n/");
501
s3_buffer->append(bucket);
502
s3_buffer->append("/");
503
s3_buffer->append(key);
504
505
#ifdef SHOW_SIGNING
506
printf("signing:\n=================\n%s\n=================\n", s3_buffer->getCString());
507
if(0){
508
const char *ptr = s3_buffer->getCString();
509
while (*ptr) {
510
printf("%x ", *ptr); ptr++;
511
}
512
printf("\n");
513
}
514
#endif
515
516
CSString *sig = signature(s3_buffer->getCString(), s3_private_key->getCString());
517
release_(s3_buffer);
518
if (headers)
519
release_(headers);
520
521
return_(sig);
522
}
523
//----------------------
524
// CURL callback functions:
525
////////////////////////////
526
//----------------------
527
static size_t send_callback(void *ptr, size_t objs, size_t obj_size, void *v_con)
528
{
529
S3ProtocolCon *con = (S3ProtocolCon*) v_con;
530
size_t data_sent, buffer_size = objs * obj_size;
531
532
if (!con->ms_data_size)
533
return 0;
534
535
enter_();
536
try_(a) {
537
data_sent = con->ms_inputStream->read((char*)ptr, buffer_size);
538
if (data_sent <= con->ms_data_size) {
539
con->ms_data_size -= data_sent;
540
if (data_sent)
541
con->ms_md5.md5_append((u_char*)ptr, data_sent); // Calculating the checksum for the data sent.
542
} else if (data_sent > con->ms_data_size)
543
CSException::RecordException(CS_CONTEXT, CS_ERR_GENERIC_ERROR, "Blob larger than expected.");
544
else if (con->ms_data_size && !data_sent)
545
CSException::RecordException(CS_CONTEXT, CS_ERR_GENERIC_ERROR, "Blob smaller than expected.");
546
}
547
catch_(a);
548
con->ms_throw_error = true;
549
data_sent = -1;
550
551
cont_(a);
552
553
return data_sent;
554
}
555
//----------------------
556
static size_t receive_data(void *vptr, size_t objs, size_t obj_size, void *v_con)
557
{
558
S3ProtocolCon *con = (S3ProtocolCon*) v_con;
559
size_t data_len = objs * obj_size;
560
561
enter_();
562
try_(a) {
563
if (con->ms_replyStatus >= 400) { // Collect the error reply.
564
if (!con->ms_errorReply)
565
con->ms_errorReply = new CSStringBuffer(50);
566
con->ms_errorReply->append((char*)vptr, data_len);
567
} else if ( con->ms_outputStream)
568
con->ms_outputStream->write((char*)vptr, data_len);
569
}
570
catch_(a);
571
con->ms_throw_error = true;
572
data_len = -1;
573
574
cont_(a);
575
return_(data_len);
576
}
577
578
#define IS_REDIRECT(s) ((s >= 300) && (s < 400))
579
//----------------------
580
static size_t receive_header(void *header, size_t objs, size_t obj_size, void *v_con)
581
{
582
S3ProtocolCon *con = (S3ProtocolCon*) v_con;
583
size_t size = objs * obj_size;
584
char *end, *ptr = (char*) header, *name, *value = NULL;
585
uint32_t name_len =0, value_len = 0;
586
587
CLOBBER_PROTECT(con);
588
CLOBBER_PROTECT(size);
589
CLOBBER_PROTECT(ptr);
590
CLOBBER_PROTECT(value);
591
CLOBBER_PROTECT(value_len);
592
CLOBBER_PROTECT(name_len);
593
594
//printf( "receive_header: %s\n", ptr);
595
end = ptr + size;
596
if (*(end -2) == '\r' && *(end -1) == '\n')
597
end -=2;
598
599
while ((end != ptr) && (*ptr == ' ')) ptr++;
600
if (end == ptr)
601
return size;
602
603
// Get the reply status.
604
// Status 100 = Continue
605
if (((!con->ms_replyStatus) || (con->ms_replyStatus == 100) || IS_REDIRECT(con->ms_replyStatus) )
606
&& !strncasecmp(ptr, "HTTP", 4)
607
) {
608
char status[4];
609
while ((end != ptr) && (*ptr != ' ')) ptr++; // skip HTTP stuff
610
while ((end != ptr) && (*ptr == ' ')) ptr++; // find the start of eh status code.
611
if (end == ptr)
612
return size;
613
614
if (end < (ptr +3)) // expecting a 3 digit status code.
615
return size;
616
617
memcpy(status, ptr, 3);
618
status[3] = 0;
619
620
con->ms_replyStatus = atoi(status);
621
}
622
623
name = ptr;
624
while ((end != ptr) && (*ptr != ':')) ptr++;
625
if (end == ptr)
626
return size;
627
name_len = ptr - name;
628
629
ptr++;
630
while ((end != ptr) && (*ptr == ' ')) ptr++;
631
if (end == ptr)
632
return size;
633
634
value = ptr;
635
value_len = end - value;
636
637
while (name[name_len-1] == ' ') name_len--;
638
while (value[value_len-1] == ' ') value_len--;
639
640
if (!strncasecmp(name, "ETag", 4)) {
641
value++; value_len -=2; // Strip quotation marks from checksum string.
642
if (value_len == HEX_CHECKSUM_VALUE_SIZE) {
643
memcpy(con->ms_s3Checksum, value, value_len);
644
con->ms_s3Checksum[value_len] = 0;
645
}
646
}
647
648
enter_();
649
try_(a) {
650
con->ms_reply_headers.addHeader(name, name_len, value, value_len);
651
}
652
653
catch_(a);
654
con->ms_throw_error = true;
655
size = (size_t)-1;
656
657
cont_(a);
658
return_(size);
659
}
660
661
//----------------------
662
663
#define SET_DATE_FROM_TIME(t, d) {strftime(d, sizeof(d), "%a, %d %b %Y %H:%M:%S GMT", gmtime(&t));}
664
#define SET_DATE(d) {time_t t = time(NULL); SET_DATE_FROM_TIME(t, d);}
665
666
bool CSS3Protocol::s3_delete(const char *bucket, const char *key)
667
{
668
CSStringBuffer *s3_buffer;
669
char date[64];
670
CSString *signed_str;
671
uint32_t retry_count = 0;
672
S3ProtocolCon *con_data;
673
674
enter_();
675
676
new_(s3_buffer, CSStringBuffer());
677
push_(s3_buffer);
678
679
new_(con_data, S3ProtocolCon());
680
push_(con_data);
681
682
retry:
683
// Clear old settings.
684
con_data->ms_reset();
685
686
SET_DATE(date);
687
688
// Build the URL
689
s3_buffer->setLength(0);
690
s3_buffer->append("http://");
691
s3_buffer->append(bucket);
692
s3_buffer->append(".");
693
s3_buffer->append(s3_server->getCString());
694
s3_buffer->append(key);
695
696
con_data->ms_setURL(s3_buffer->getCString());
697
698
// Add the 'DATE' header
699
s3_buffer->setLength(0);
700
s3_buffer->append("Date: ");
701
s3_buffer->append(date);
702
con_data->ms_setHeader(s3_buffer->getCString());
703
704
// Create the authentication signature and add the 'Authorization' header
705
signed_str = s3_getSignature("DELETE", NULL, NULL, date, bucket, key);
706
push_(signed_str);
707
s3_buffer->setLength(0);
708
s3_buffer->append("Authorization: AWS ");
709
s3_buffer->append(s3_public_key->getCString());
710
s3_buffer->append(":");
711
s3_buffer->append(signed_str->getCString());
712
release_(signed_str); signed_str = NULL;
713
714
con_data->ms_setHeader(s3_buffer->getCString());
715
716
con_data->ms_execute_delete_request();
717
718
if (con_data->ms_retry) {
719
if (retry_count == s3_maxRetrys) {
720
CSException::throwException(CS_CONTEXT, CS_ERR_GENERIC_ERROR, "S3 operation aborted after max retries.");
721
}
722
printf("RETRY: s3_delete()\n");
723
retry_count++;
724
goto retry;
725
}
726
727
bool notFound = con_data->ms_notFound;
728
release_(con_data);
729
release_(s3_buffer);
730
731
return_(!notFound);
732
}
733
734
//-------------------------------
735
void CSS3Protocol::s3_copy(const char *dest_server, const char *dest_bucket, const char *dest_key, const char *src_bucket, const char *src_key)
736
{
737
CSStringBuffer *s3_buffer;
738
char date[64];
739
CSString *signed_str;
740
uint32_t retry_count = 0;
741
S3ProtocolCon *con_data;
742
743
enter_();
744
745
new_(s3_buffer, CSStringBuffer());
746
push_(s3_buffer);
747
748
new_(con_data, S3ProtocolCon());
749
push_(con_data);
750
751
if (!dest_server)
752
dest_server = s3_server->getCString();
753
754
retry:
755
// Clear old settings.
756
con_data->ms_reset();
757
758
SET_DATE(date);
759
760
// Build the URL
761
s3_buffer->setLength(0);
762
s3_buffer->append("http://");
763
s3_buffer->append(dest_bucket);
764
s3_buffer->append(".");
765
s3_buffer->append(s3_server->getCString());
766
s3_buffer->append(dest_key);
767
768
con_data->ms_setURL(s3_buffer->getCString());
769
770
// Add the destination location
771
s3_buffer->setLength(0);
772
s3_buffer->append("Host: ");
773
s3_buffer->append(dest_bucket);
774
s3_buffer->append(".");
775
s3_buffer->append(dest_server);
776
s3_buffer->setLength(s3_buffer->length() -1); // trim the '/'
777
con_data->ms_setHeader(s3_buffer->getCString());
778
779
// Add the source location
780
s3_buffer->setLength(0);
781
s3_buffer->append("x-amz-copy-source:");
782
s3_buffer->append(src_bucket);
783
s3_buffer->append("/");
784
s3_buffer->append(src_key);
785
con_data->ms_setHeader(s3_buffer->getCString());
786
787
// Create the authentication signature and add the 'Authorization' header
788
signed_str = s3_getSignature("PUT", NULL, NULL, date, dest_bucket, dest_key, CSString::newString(s3_buffer->getCString()));
789
push_(signed_str);
790
791
// Add the 'DATE' header
792
s3_buffer->setLength(0);
793
s3_buffer->append("Date: ");
794
s3_buffer->append(date);
795
con_data->ms_setHeader(s3_buffer->getCString());
796
797
// Add the signature
798
s3_buffer->setLength(0);
799
s3_buffer->append("Authorization: AWS ");
800
s3_buffer->append(s3_public_key->getCString());
801
s3_buffer->append(":");
802
s3_buffer->append(signed_str->getCString());
803
release_(signed_str); signed_str = NULL;
804
con_data->ms_setHeader(s3_buffer->getCString());
805
806
con_data->ms_execute_copy_request();
807
808
if (con_data->ms_notFound) {
809
s3_buffer->setLength(0);
810
s3_buffer->append("Cloud copy failed, object not found: ");
811
s3_buffer->append(src_bucket);
812
s3_buffer->append(" ");
813
s3_buffer->append(src_key);
814
CSException::throwException(CS_CONTEXT, CS_ERR_GENERIC_ERROR, s3_buffer->getCString());
815
}
816
817
if (con_data->ms_retry) {
818
if (retry_count == s3_maxRetrys) {
819
CSException::throwException(CS_CONTEXT, CS_ERR_GENERIC_ERROR, "S3 operation aborted after max retries.");
820
}
821
printf("RETRY: s3_copy()\n");
822
retry_count++;
823
goto retry;
824
}
825
826
release_(con_data);
827
release_(s3_buffer);
828
829
exit_();
830
}
831
832
833
//-------------------------------
834
CSVector *CSS3Protocol::s3_receive(CSOutputStream *output, const char *bucket, const char *key, bool *found)
835
{
836
CSStringBuffer *s3_buffer;
837
char date[64];
838
CSString *signed_str;
839
uint32_t retry_count = 0;
840
S3ProtocolCon *con_data;
841
CSVector *replyHeaders;
842
843
enter_();
844
845
push_(output);
846
847
new_(s3_buffer, CSStringBuffer());
848
push_(s3_buffer);
849
850
new_(con_data, S3ProtocolCon());
851
push_(con_data);
852
853
retry:
854
// Clear old settings.
855
con_data->ms_reset();
856
857
SET_DATE(date);
858
859
// Build the URL
860
s3_buffer->setLength(0);
861
s3_buffer->append("http://");
862
s3_buffer->append(bucket);
863
s3_buffer->append(".");
864
s3_buffer->append(s3_server->getCString());
865
s3_buffer->append(key);
866
867
con_data->ms_setURL(s3_buffer->getCString());
868
869
// Add the 'DATE' header
870
s3_buffer->setLength(0);
871
s3_buffer->append("Date: ");
872
s3_buffer->append(date);
873
con_data->ms_setHeader(s3_buffer->getCString());
874
875
// Create the authentication signature and add the 'Authorization' header
876
signed_str = s3_getSignature("GET", NULL, NULL, date, bucket, key);
877
push_(signed_str);
878
s3_buffer->setLength(0);
879
s3_buffer->append("Authorization: AWS ");
880
s3_buffer->append(s3_public_key->getCString());
881
s3_buffer->append(":");
882
s3_buffer->append(signed_str->getCString());
883
release_(signed_str); signed_str = NULL;
884
con_data->ms_setHeader(s3_buffer->getCString());
885
886
con_data->ms_execute_get_request(RETAIN(output));
887
888
if (con_data->ms_retry) {
889
if (retry_count == s3_maxRetrys) {
890
CSException::throwException(CS_CONTEXT, CS_ERR_GENERIC_ERROR, "S3 operation aborted after max retries.");
891
}
892
printf("RETRY: s3_receive()\n");
893
retry_count++;
894
output->reset();
895
goto retry;
896
}
897
898
*found = !con_data->ms_notFound;
899
replyHeaders = con_data->ms_reply_headers.takeHeaders();
900
release_(con_data);
901
release_(s3_buffer);
902
release_(output);
903
904
return_(replyHeaders);
905
}
906
907
class S3ListParser : public CSXMLBuffer {
908
909
CSVector *list;
910
public:
911
912
913
bool parseListData(const char *data, size_t len, CSVector *keys)
914
{
915
list = keys;
916
return parseData(data, len, 0);
917
}
918
919
private:
920
virtual bool openNode(char *path, char *value) {
921
if (value && *value && (strcmp(path,"/listbucketresult/contents/key/") == 0))
922
list->add(CSString::newString(value));
923
return true;
924
}
925
926
virtual bool closeNode(char *path) {
927
(void)path;
928
return true;
929
}
930
931
virtual bool addAttribute(char *path, char *name, char *value) {
932
(void)path;
933
(void)name;
934
(void)value;
935
return true;
936
}
937
938
};
939
940
//-------------------------------
941
static CSVector *parse_s3_list(CSMemoryOutputStream *output)
942
{
943
S3ListParser s3ListParser;
944
const char *data;
945
CSVector *vector;
946
size_t len;
947
948
enter_();
949
950
push_(output);
951
952
new_(vector, CSVector(10));
953
push_(vector);
954
955
data = (const char *) output->getMemory(&len);
956
if (!s3ListParser.parseListData(data, len, vector)) {
957
int err;
958
char *msg;
959
960
s3ListParser.getError(&err, &msg);
961
CSException::throwException(CS_CONTEXT, CS_ERR_GENERIC_ERROR, msg);
962
}
963
964
pop_(vector);
965
release_(output);
966
return_(vector);
967
}
968
969
970
//-------------------------------
971
CSVector *CSS3Protocol::s3_list(const char *bucket, const char *key_prefix, uint32_t max)
972
{
973
CSStringBuffer *s3_buffer;
974
char date[64];
975
CSString *signed_str;
976
CSMemoryOutputStream *output;
977
uint32_t retry_count = 0;
978
S3ProtocolCon *con_data;
979
enter_();
980
981
new_(s3_buffer, CSStringBuffer());
982
push_(s3_buffer);
983
984
output = CSMemoryOutputStream::newStream(1024, 1024);
985
push_(output);
986
987
new_(con_data, S3ProtocolCon());
988
push_(con_data);
989
990
retry:
991
992
// Clear old settings.
993
con_data->ms_reset();
994
995
SET_DATE(date);
996
997
// Build the URL
998
s3_buffer->setLength(0);
999
s3_buffer->append("http://");
1000
s3_buffer->append(bucket);
1001
s3_buffer->append(".");
1002
s3_buffer->append(s3_server->getCString());
1003
if (key_prefix) {
1004
s3_buffer->append("?prefix=");
1005
s3_buffer->append(key_prefix);
1006
}
1007
1008
if (max) {
1009
if (key_prefix)
1010
s3_buffer->append("&max-keys=");
1011
else
1012
s3_buffer->append("?max-keys=");
1013
s3_buffer->append(max);
1014
}
1015
1016
con_data->ms_setURL(s3_buffer->getCString());
1017
1018
// Add the 'DATE' header
1019
s3_buffer->setLength(0);
1020
s3_buffer->append("Date: ");
1021
s3_buffer->append(date);
1022
con_data->ms_setHeader(s3_buffer->getCString());
1023
1024
// Create the authentication signature and add the 'Authorization' header
1025
signed_str = s3_getSignature("GET", NULL, NULL, date, bucket, "");
1026
push_(signed_str);
1027
s3_buffer->setLength(0);
1028
s3_buffer->append("Authorization: AWS ");
1029
s3_buffer->append(s3_public_key->getCString());
1030
s3_buffer->append(":");
1031
s3_buffer->append(signed_str->getCString());
1032
release_(signed_str); signed_str = NULL;
1033
con_data->ms_setHeader(s3_buffer->getCString());
1034
1035
con_data->ms_execute_get_request(RETAIN(output));
1036
1037
if (con_data->ms_retry) {
1038
if (retry_count == s3_maxRetrys) {
1039
CSException::throwException(CS_CONTEXT, CS_ERR_GENERIC_ERROR, "S3 operation aborted after max retries.");
1040
}
1041
printf("RETRY: s3_list()\n");
1042
retry_count++;
1043
output->reset();
1044
goto retry;
1045
}
1046
1047
release_(con_data);
1048
pop_(output);
1049
release_(s3_buffer);
1050
return_(parse_s3_list(output));
1051
}
1052
1053
//-------------------------------
1054
CSString *CSS3Protocol::s3_getAuthorization(const char *bucket, const char *key, const char *content_type, uint32_t *s3AuthorizationTime)
1055
{
1056
char date[64];
1057
CSString *signed_str;
1058
time_t sys_time;
1059
1060
enter_();
1061
1062
if (!content_type)
1063
content_type = "binary/octet-stream";
1064
1065
sys_time = time(NULL);
1066
1067
*s3AuthorizationTime = (uint32_t)sys_time;
1068
1069
SET_DATE_FROM_TIME(sys_time, date);
1070
signed_str = s3_getSignature("PUT", NULL, content_type, date, bucket, key);
1071
return_(signed_str);
1072
}
1073
1074
//-------------------------------
1075
CSVector *CSS3Protocol::s3_send(CSInputStream *input, const char *bucket, const char *key, off64_t size, const char *content_type, Md5Digest *digest, const char *s3Authorization, time_t s3AuthorizationTime)
1076
{
1077
CSStringBuffer *s3_buffer;
1078
char date[64];
1079
CSString *signed_str;
1080
Md5Digest dummy_digest;
1081
uint32_t retry_count = 0;
1082
S3ProtocolCon *con_data;
1083
CSVector *replyHeaders;
1084
1085
enter_();
1086
push_(input);
1087
1088
new_(s3_buffer, CSStringBuffer());
1089
push_(s3_buffer);
1090
1091
new_(con_data, S3ProtocolCon());
1092
push_(con_data);
1093
1094
if (!digest)
1095
digest = &dummy_digest;
1096
1097
if (!content_type)
1098
content_type = "binary/octet-stream";
1099
1100
retry:
1101
1102
// Clear old settings.
1103
con_data->ms_reset();
1104
1105
if (s3Authorization) {
1106
SET_DATE_FROM_TIME(s3AuthorizationTime, date);
1107
} else {
1108
SET_DATE(date);
1109
}
1110
1111
// Build the URL
1112
s3_buffer->setLength(0);
1113
s3_buffer->append("http://");
1114
s3_buffer->append(bucket);
1115
s3_buffer->append(".");
1116
s3_buffer->append(s3_server->getCString());
1117
s3_buffer->append(key);
1118
1119
con_data->ms_setURL(s3_buffer->getCString());
1120
1121
// Add the 'DATE' header
1122
s3_buffer->setLength(0);
1123
s3_buffer->append("Date: ");
1124
s3_buffer->append(date);
1125
con_data->ms_setHeader(s3_buffer->getCString());
1126
1127
// Add the 'Content-Type' header
1128
s3_buffer->setLength(0);
1129
s3_buffer->append("Content-Type: ");
1130
s3_buffer->append(content_type);
1131
con_data->ms_setHeader(s3_buffer->getCString());
1132
1133
// Create the authentication signature and add the 'Authorization' header
1134
if (!s3Authorization)
1135
signed_str = s3_getSignature("PUT", NULL, content_type, date, bucket, key);
1136
else
1137
signed_str = CSString::newString(s3Authorization);
1138
push_(signed_str);
1139
s3_buffer->setLength(0);
1140
s3_buffer->append("Authorization: AWS ");
1141
s3_buffer->append(s3_public_key->getCString());
1142
s3_buffer->append(":");
1143
s3_buffer->append(signed_str->getCString());
1144
release_(signed_str); signed_str = NULL;
1145
con_data->ms_setHeader(s3_buffer->getCString());
1146
1147
con_data->ms_execute_put_request(RETAIN(input), size, digest);
1148
1149
if (con_data->ms_retry) {
1150
if (retry_count == s3_maxRetrys) {
1151
CSException::throwException(CS_CONTEXT, CS_ERR_GENERIC_ERROR, "S3 operation aborted after max retries.");
1152
}
1153
printf("RETRY: s3_send()\n");
1154
retry_count++;
1155
input->reset();
1156
goto retry;
1157
}
1158
1159
replyHeaders = con_data->ms_reply_headers.takeHeaders();
1160
release_(con_data);
1161
release_(s3_buffer);
1162
release_(input);
1163
return_(replyHeaders);
1164
}
1165
1166
//-------------------------------
1167
CSString *CSS3Protocol::s3_getDataURL(const char *bucket, const char *key, uint32_t keep_alive)
1168
{
1169
CSStringBuffer *s3_buffer;
1170
char timeout[32];
1171
CSString *signed_str;
1172
enter_();
1173
1174
new_(s3_buffer, CSStringBuffer());
1175
push_(s3_buffer);
1176
1177
snprintf(timeout, 32, "%"PRId32"", ((uint32_t)time(NULL)) + keep_alive);
1178
1179
signed_str = s3_getSignature("GET", NULL, NULL, timeout, bucket, key);
1180
//printf("Unsafe: \"%s\"\n", signed_str->getCString());
1181
signed_str = urlEncode(signed_str); // Because the signature is in the URL it must be URL encoded.
1182
//printf(" Safe: \"%s\"\n", signed_str->getCString());
1183
push_(signed_str);
1184
1185
s3_buffer->setLength(0);
1186
s3_buffer->append("http://");
1187
s3_buffer->append(bucket);
1188
s3_buffer->append(".");
1189
s3_buffer->append(s3_server->getCString());
1190
s3_buffer->append(key);
1191
1192
s3_buffer->append("?AWSAccessKeyId=");
1193
s3_buffer->append(s3_public_key->getCString());
1194
s3_buffer->append("&Expires=");
1195
s3_buffer->append(timeout);
1196
s3_buffer->append("&Signature=");
1197
s3_buffer->append(signed_str->getCString());
1198
1199
release_(signed_str);
1200
1201
pop_(s3_buffer);
1202
CSString *str = CSString::newString(s3_buffer);
1203
return_(str);
1204
}
1205
1206
//#define S3_UNIT_TEST
1207
#ifdef S3_UNIT_TEST
1208
static void show_help_info(const char *cmd)
1209
{
1210
printf("Get authenticated query string:\n\t%s q <bucket> <object_key> <timeout>\n", cmd);
1211
printf("Delete object:\n\t%s d <bucket> <object_key>\n", cmd);
1212
printf("Delete all object with a given prefix:\n\t%s D <bucket> <object_prefix>\n", cmd);
1213
printf("Get object, data will be written to 'prottest.out':\n\t%s g <bucket> <object_key> <timeout>\n", cmd);
1214
printf("Put (Upload) an object:\n\t%s p <bucket> <object_key> <file>\n", cmd);
1215
printf("List objects in the bucket:\n\t%s l <bucket> [<object_prefix> [max_list_size]]\n", cmd);
1216
printf("Copy object:\n\t%s c <src_bucket> <src_object_key> <dst_bucket> <dst_object_key> \n", cmd);
1217
printf("Copy all object with a given prefix:\n\t%s C <src_bucket> <object_key_prefix> <dst_bucket> \n", cmd);
1218
}
1219
1220
void dump_headers(CSVector *header_array)
1221
{
1222
CSHTTPHeaders headers;
1223
1224
headers.setHeaders(header_array);
1225
printf("Reply Headers:\n");
1226
printf("--------------\n");
1227
1228
for (uint32_t i = 0; i < headers.numHeaders(); i++) {
1229
CSHeader *h = headers.getHeader(i);
1230
1231
printf("%s : %s\n", h->getNameCString(), h->getValueCString());
1232
h->release();
1233
}
1234
printf("--------------\n");
1235
headers.clearHeaders();
1236
}
1237
1238
int main(int argc, char **argv)
1239
{
1240
CSThread *main_thread;
1241
const char *pub_key;
1242
const char *priv_key;
1243
CSS3Protocol *prot = NULL;
1244
1245
if (argc < 3) {
1246
show_help_info(argv[0]);
1247
return 0;
1248
}
1249
1250
if (! CSThread::startUp()) {
1251
CSException::throwException(CS_CONTEXT, ENOMEM, "CSThread::startUp() failed.");
1252
return 1;
1253
}
1254
1255
cs_init_memory();
1256
1257
main_thread = new CSThread( NULL);
1258
CSThread::setSelf(main_thread);
1259
1260
enter_();
1261
try_(a) {
1262
1263
pub_key = getenv("S3_ACCESS_KEY_ID");
1264
priv_key = getenv("S3_SECRET_ACCESS_KEY");
1265
new_(prot, CSS3Protocol());
1266
push_(prot);
1267
1268
prot->s3_setServer("s3.amazonaws.com/");
1269
prot->s3_setPublicKey(pub_key);
1270
prot->s3_setPrivateKey(priv_key);
1271
1272
switch (argv[1][0]) {
1273
case 'q': // Get the query string
1274
if (argc == 5) {
1275
CSString *qstr = prot->s3_getDataURL(argv[2], argv[3], atoi(argv[4]));
1276
printf("To test call:\ncurl -L -D - \"%s\"\n", qstr->getCString());
1277
qstr->release();
1278
} else
1279
printf("Bad command: q <bucket> <object_key> <timeout>\n");
1280
1281
break;
1282
case 'd': // Delete the object
1283
if (argc == 4) {
1284
printf("delete %s %s\n", argv[2], argv[3]);
1285
if (!prot->s3_delete(argv[2], argv[3]))
1286
printf("%s/%s could not be found.\n", argv[2], argv[3]);
1287
1288
} else
1289
printf("Bad command: d <bucket> <object_key>\n");
1290
1291
break;
1292
case 'D': // Delete objects like
1293
if (argc == 4) {
1294
CSVector *list;
1295
CSString *key;
1296
1297
list = prot->s3_list(argv[2], argv[3]);
1298
push_(list);
1299
while (key = (CSString*) list->take(0)) {
1300
printf("Deleting %s\n", key->getCString());
1301
prot->s3_delete(argv[2], key->getCString());
1302
key->release();
1303
}
1304
release_(list);
1305
1306
} else
1307
printf("Bad command: D <bucket> <object_key_prefix>\n");
1308
1309
break;
1310
case 'g': // Get the object
1311
if (argc == 4) {
1312
CSFile *output;
1313
CSVector *headers;
1314
bool found;
1315
1316
output = CSFile::newFile("prottest.out");
1317
push_(output);
1318
output->open(CSFile::CREATE | CSFile::TRUNCATE);
1319
headers = prot->s3_receive(output->getOutputStream(), argv[2], argv[3], &found);
1320
if (!found)
1321
printf("%s/%s could not be found.\n", argv[2], argv[3]);
1322
1323
dump_headers(headers);
1324
1325
release_(output);
1326
} else
1327
printf("Bad command: g <bucket> <object_key>\n");
1328
1329
break;
1330
1331
case 'p': // Put (Upload) the object
1332
if (argc == 5) {
1333
CSFile *input;
1334
CSVector *headers;
1335
1336
input = CSFile::newFile(argv[4]);
1337
push_(input);
1338
input->open(CSFile::READONLY);
1339
headers = prot->s3_send(input->getInputStream(), argv[2], argv[3], input->myFilePath->getSize());
1340
dump_headers(headers);
1341
release_(input);
1342
} else
1343
printf("Bad command: p <bucket> <object_key> <file> \n");
1344
1345
break;
1346
1347
case 'c': // Copy the object
1348
if (argc == 6) {
1349
prot->s3_copy(NULL, argv[4], argv[5], argv[2], argv[3]);
1350
} else
1351
printf("Bad command: c <src_bucket> <src_object_key> <dst_bucket> <dst_object_key>\n");
1352
1353
break;
1354
1355
case 'C': // Copy objects like
1356
if (argc == 5) {
1357
CSVector *list;
1358
CSString *key;
1359
1360
list = prot->s3_list(argv[2], argv[3]);
1361
push_(list);
1362
while (key = (CSString*) list->take(0)) {
1363
printf("Copying %s\n", key->getCString());
1364
prot->s3_copy(NULL, argv[4], key->getCString(), argv[2], key->getCString());
1365
key->release();
1366
}
1367
release_(list);
1368
1369
} else
1370
printf("Bad command: C <src_bucket> <object_key_prefix> <dst_bucket>\n");
1371
1372
break;
1373
case 'l': // List the object
1374
if ((argc == 3) || (argc == 4) || (argc == 5)) {
1375
uint32_t max = 0;
1376
char *prefix = NULL;
1377
CSVector *list;
1378
CSString *key;
1379
1380
if (argc > 3) {
1381
prefix = argv[3];
1382
if (!strlen(prefix))
1383
prefix = NULL;
1384
}
1385
1386
if (argc == 5)
1387
max = atol(argv[4]);
1388
1389
list = prot->s3_list(argv[2], prefix, max);
1390
push_(list);
1391
while (key = (CSString*) list->take(0)) {
1392
printf("%s\n", key->getCString());
1393
key->release();
1394
}
1395
release_(list);
1396
1397
} else
1398
printf("Bad command: l <bucket> [<object_prefix> [max_list_size]] \n");
1399
1400
break;
1401
default:
1402
printf("Unknown command.\n");
1403
show_help_info(argv[0]);
1404
}
1405
1406
release_(prot);
1407
}
1408
1409
catch_(a);
1410
self->logException();
1411
1412
cont_(a);
1413
1414
outer_()
1415
main_thread->release();
1416
cs_exit_memory();
1417
CSThread::shutDown();
1418
return 0;
1419
}
1420
1421
#endif
1422
1423
Loggerhead is a web-based interface for Breezy
Version: 2.0.1