« back to all changes in this revision
Viewing changes to plugin/pbxt/src/iotest_xt.c
- browse files at revision 1462
- compare with another revision
- compare with revision 38
- download diff
- download tarball
- view history from revision 1462
- Committer: Brian Aker
- Date: 2010-04-12 07:43:55 UTC
- mfrom: (1455.3.13 drizzle-pbxt-6)
- Revision ID: brian@gaz-20100412074355-udi9dwjlcnmz0oz6
Merge PBXT
- files added:
- plugin/pbxt
- plugin/pbxt/bin
- plugin/pbxt/src
- tests/r/pbxt
added
removed
plugin/pbxt/src/iotest_xt.c
1
/* Copyright (c) 2005 PrimeBase Technologies GmbH
2
*
3
* PrimeBase XT
4
*
5
* This program is free software; you can redistribute it and/or modify
6
* it under the terms of the GNU General Public License as published by
7
* the Free Software Foundation; either version 2 of the License, or
8
* (at your option) any later version.
9
*
10
* This program is distributed in the hope that it will be useful,
11
* but WITHOUT ANY WARRANTY; without even the implied warranty of
12
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13
* GNU General Public License for more details.
14
*
15
* You should have received a copy of the GNU General Public License
16
* along with this program; if not, write to the Free Software
17
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
18
*
19
* 2009-07-30 Paul McCullagh
20
*
21
* H&G2JCtL
22
*/
23
24
#include <pthread.h>
25
26
27
#include <stdlib.h>
28
#include <stddef.h>
29
#include <stdio.h>
30
#include <sys/time.h>
31
#include <unistd.h>
32
#include <dirent.h>
33
#include <stdio.h>
34
#include <sys/stat.h>
35
#include <fcntl.h>
36
#include <sys/types.h>
37
#include <ctype.h>
38
#include <string.h>
39
#include <errno.h>
40
#include <limits.h>
41
#include <assert.h>
42
43
#ifdef __APPLE__
44
#define MAC
45
#endif
46
47
/*
48
* Define this if I/O should pause.
49
*/
50
//#define SHOULD_PAUSE
51
//#define PERIODIC_FLUSH
52
53
#define SIM_RECORD_SIZE 221
54
#ifdef MAC
55
#define SIM_FILE_SIZE (256*1024*1024)
56
#define SIM_WRITE_AMOUNT (1*1024*1024)
57
//#define SIM_WRITE_AMOUNT (221 * 100)
58
#else
59
#define SIM_FILE_SIZE ((off_t) (2L*1024L*1024L*1024L))
60
#define SIM_WRITE_AMOUNT (8*1024*1024)
61
#endif
62
#define SIM_FLUSH_THRESHOLD (2*1024*1024)
63
#define SIM_PAUSE_THRESHOLD (10*1024*1024)
64
65
#ifndef SHOULD_PAUSE
66
#undef SIM_PAUSE_THRESHOLD
67
#define SIM_PAUSE_THRESHOLD 0
68
#endif
69
70
#ifndef PERIODIC_FLUSH
71
#undef SIM_FLUSH_THRESHOLD
72
#define SIM_FLUSH_THRESHOLD 0
73
#endif
74
75
#define my_time unsigned long long
76
#define u_long unsigned long
77
#define TRUE 1
78
#define FALSE 0
79
80
typedef struct SortedRec {
81
off_t sr_offset;
82
int sr_order;
83
char *sr_data;
84
} SortedRec;
85
86
#define SORTED_MAX_RECORDS 10000
87
#define SORTED_DATA_SIZE (SORTED_MAX_RECORDS * SIM_RECORD_SIZE)
88
#define SORTED_BUFFER_SIZE (256*1024)
89
90
typedef struct RewriteRec {
91
off_t rr_offset;
92
off_t rr_size;
93
} RewriteRec;
94
95
#define REWRITE_MAX_RECORDS 1000
96
/* This is the maximum distance between to blocks that
97
* will cause the blocks to be combined and written
98
* as one block!
99
*/
100
#ifdef MAC
101
#define REWRITE_BLOCK_DISTANCE (64*1024)
102
#else
103
#define REWRITE_BLOCK_DISTANCE (1024*1024)
104
#endif
105
106
#define REWRITE_RECORD_LIMIT 256
107
108
typedef struct File {
109
int file_fh;
110
111
int fi_monitor_index;
112
char fi_file_path[200];
113
char fi_test_name[200];
114
char fi_monitor_name[10];
115
int fi_monitor_active;
116
117
my_time total_time;
118
119
my_time flush_start;
120
my_time flush_time;
121
u_long flush_count;
122
123
my_time last_flush_time;
124
u_long last_flush_count;
125
126
my_time write_start;
127
my_time write_time;
128
u_long write_count;
129
130
off_t last_block_offset;
131
size_t last_block_size;
132
u_long block_write_count;
133
134
my_time last_write_time;
135
u_long last_write_count;
136
137
my_time read_start;
138
my_time read_time;
139
u_long read_count;
140
141
my_time last_read_time;
142
u_long last_read_count;
143
144
/* Sorted file I/O */
145
int sf_rec_count;
146
SortedRec sf_records[SORTED_MAX_RECORDS];
147
size_t sf_alloc_pos;
148
int sf_order;
149
char sf_data[SORTED_DATA_SIZE];
150
char sf_buffer[SORTED_BUFFER_SIZE];
151
152
/* Re-write sync: */
153
off_t rs_min_block_offset;
154
off_t rs_max_block_offset;
155
size_t rs_flush_block_total;
156
size_t rs_rec_count;
157
RewriteRec rs_records[REWRITE_MAX_RECORDS];
158
159
void (*fi_write)(struct File *f, void *block, size_t size, off_t start);
160
void (*fi_sync)(struct File *f);
161
void (*fi_write_all)(struct File *f);
162
} File;
163
164
/* ------------------- TIMING ------------------- */
165
166
static my_time my_clock(void)
167
{
168
static my_time my_start_clock = 0;
169
struct timeval tv;
170
my_time now;
171
172
gettimeofday(&tv, NULL);
173
now = (my_time) tv.tv_sec * (my_time) 1000000 + tv.tv_usec;
174
if (my_start_clock)
175
return now - my_start_clock;
176
my_start_clock = now;
177
return 0;
178
}
179
180
/* ------------------- ERRORS ------------------- */
181
182
static void print_error(char *file, int err)
183
{
184
printf("ERROR %s: %s\n", file, strerror(err));
185
}
186
187
static void error_exit(char *file, int err)
188
{
189
print_error(file, err);
190
exit(1);
191
}
192
193
static void fatal_error(char *message)
194
{
195
printf("%s", message);
196
exit(1);
197
}
198
199
/* ------------------- ERRORS ------------------- */
200
201
static void *my_bsearch(void *key, register const void *base, size_t count, size_t size, size_t *idx, int (*compare)(void *key, void *rec))
202
{
203
register size_t i;
204
register size_t guess;
205
register int r;
206
207
i = 0;
208
while (i < count) {
209
guess = (i + count - 1) >> 1;
210
r = compare(key, ((char *) base) + guess * size);
211
if (r == 0) {
212
*idx = guess;
213
return ((char *) base) + guess * size;
214
}
215
if (r < 0)
216
count = guess;
217
else
218
i = guess + 1;
219
}
220
221
*idx = i;
222
return NULL;
223
}
224
225
/* ------------------- MONITORING THREAD ------------------- */
226
227
/* Monitor file types: */
228
#define MAX_FILES 10
229
230
static int monitor_file_count;
231
static int monitor_files_in_use;
232
static int monitor_running;
233
static File *monitor_files[MAX_FILES];
234
235
static void monitor_file(File *f, const char *test_name, const char *monitor_name)
236
{
237
if (monitor_file_count == MAX_FILES)
238
fatal_error("Too many files to monitor");
239
monitor_files[monitor_file_count] = f;
240
f->fi_monitor_index = monitor_file_count;
241
sprintf(f->fi_file_path, "test-data-%d", (int) monitor_file_count);
242
strcpy(f->fi_test_name, test_name);
243
strcpy(f->fi_monitor_name, monitor_name);
244
monitor_file_count++;
245
f->fi_monitor_active = TRUE;
246
monitor_files_in_use++;
247
}
248
249
static void unmonitor_file(File *f)
250
{
251
/* Wait for the last activity to be reported! */
252
while (monitor_running && f->fi_monitor_active) {
253
usleep(100);
254
}
255
monitor_files_in_use--;
256
monitor_files[f->fi_monitor_index] = NULL;
257
}
258
259
static void print_header()
260
{
261
File *f;
262
int i;
263
264
printf("time ");
265
for (i=0; i<MAX_FILES; i++) {
266
if ((f = monitor_files[i])) {
267
printf("%7s %5s %9s %5s %9s %5s ", f->fi_monitor_name, f->fi_monitor_name, f->fi_monitor_name, f->fi_monitor_name, f->fi_monitor_name, f->fi_monitor_name);
268
}
269
}
270
printf("\n");
271
printf(" ");
272
for (i=0; i<MAX_FILES; i++) {
273
if ((f = monitor_files[i])) {
274
printf("%7s %5s %9s %5s %9s %5s ", "flush", "ftime", "write", "wtime", "read", "rtime");
275
}
276
}
277
printf("\n");
278
}
279
280
static void *iotest_monitor(void *data)
281
{
282
File *f;
283
int i;
284
int row = 0;
285
my_time curr, last;
286
my_time now, fstart, wstart, rstart;
287
int version = 0;
288
int activity;
289
290
my_time curr_flush_time;
291
u_long curr_flush_count;
292
my_time curr_write_time;
293
u_long curr_write_count;
294
my_time curr_read_time;
295
u_long curr_read_count;
296
297
my_time flush_time;
298
u_long flush_count;
299
my_time write_time;
300
u_long write_count;
301
my_time read_time;
302
u_long read_count;
303
304
monitor_running = TRUE;
305
last = my_clock();
306
for (;;) {
307
curr = my_clock();
308
309
if (!monitor_files_in_use)
310
goto wait_phase;
311
312
if ((row % 20) == 0 || version != monitor_file_count) {
313
version = monitor_file_count;
314
print_header();
315
}
316
317
printf("%4.0f ", ((double) curr - (double) last) / (double) 1000);
318
activity = FALSE;
319
for (i=0; i<MAX_FILES; i++) {
320
if ((f = monitor_files[i])) {
321
curr_flush_time = f->flush_time;
322
fstart = f->flush_start;
323
curr_write_time = f->write_time;
324
wstart = f->write_start;
325
curr_read_time = f->read_time;
326
rstart = f->read_start;
327
now = my_clock();
328
329
if (fstart)
330
curr_flush_time += now - fstart;
331
flush_time = curr_flush_time - f->last_flush_time;
332
f->last_flush_time = curr_flush_time;
333
334
curr_flush_count = f->flush_count;
335
flush_count = curr_flush_count - f->last_flush_count;
336
f->last_flush_count = curr_flush_count;
337
338
if (wstart)
339
curr_write_time += now - wstart;
340
write_time = curr_write_time - f->last_write_time;
341
f->last_write_time = curr_write_time;
342
343
curr_write_count = f->write_count;
344
write_count = curr_write_count - f->last_write_count;
345
f->last_write_count = curr_write_count;
346
347
if (rstart)
348
curr_read_time += now - rstart;
349
read_time = curr_read_time - f->last_read_time;
350
f->last_read_time = curr_read_time;
351
352
curr_read_count = f->read_count;
353
read_count = curr_read_count - f->last_read_count;
354
f->last_read_count = curr_read_count;
355
356
printf("%7lu %5.0f %9.2f %5.0f %9.2f %5.0f ", flush_count, (double) flush_time / (double) 1000,
357
(double) write_count / (double) 1024, (double) write_time / (double) 1000,
358
(double) read_count / (double) 1024, (double) read_time / (double) 1000);
359
if (flush_count || flush_time || write_count || write_time || read_count || read_time) {
360
f->fi_monitor_active = TRUE;
361
activity = TRUE;
362
}
363
else
364
f->fi_monitor_active = FALSE;
365
}
366
}
367
printf("\n");
368
row++;
369
370
wait_phase:
371
372
/* Leave the loop, only when there is no more activity. */
373
if (!monitor_running && !activity)
374
break;
375
376
do {
377
usleep(1000);
378
} while (my_clock() - curr < 1000000);
379
last = curr;
380
}
381
382
return NULL;
383
}
384
385
/* ------------------- BASIC FILE I/O ------------------- */
386
387
#define PREFILL 0
388
#define SET_EOF 1
389
#define TRUNCATE 2
390
391
#define XT_MASK ((S_IRUSR | S_IWUSR) | (S_IRGRP | S_IWGRP) | (S_IROTH))
392
393
static void create_file(File *f, size_t size, int type)
394
{
395
int fd;
396
size_t tfer;
397
char *block;
398
size_t i;
399
off_t eof;
400
off_t offset;
401
402
if (!(block = (char *) malloc(512)))
403
error_exit(f->fi_file_path, errno);
404
for (i=0; i<512; i++)
405
block[i] = (char) i;
406
407
fd = open(f->fi_file_path, O_CREAT | O_RDWR, XT_MASK);
408
if (fd == -1)
409
error_exit(f->fi_file_path, errno);
410
411
eof = lseek(fd, 0, SEEK_END);
412
if (type == PREFILL && size == eof)
413
goto done;
414
415
if (ftruncate(fd, 0) == -1)
416
error_exit(f->fi_file_path, errno);
417
418
if (type == SET_EOF) {
419
if (size > 512)
420
offset = size - 512;
421
else
422
offset = 0;
423
if (pwrite(fd, block, 512, offset) != 512)
424
error_exit(f->fi_file_path, errno);
425
}
426
else {
427
offset = 0;
428
while (size > 0) {
429
tfer = size;
430
if (tfer > 512)
431
tfer = 512;
432
if (pwrite(fd, block, tfer, offset) != tfer)
433
error_exit(f->fi_file_path, errno);
434
size -= tfer;
435
offset += tfer;
436
}
437
}
438
439
if (fsync(fd) == -1)
440
error_exit(f->fi_file_path, errno);
441
442
done:
443
close(fd);
444
free(block);
445
}
446
447
/*
448
static void delete_file(char *file)
449
{
450
unlink(file);
451
}
452
*/
453
454
static void write_file(File *f, void *block, size_t size, off_t offset)
455
{
456
my_time t, s;
457
458
s = my_clock();
459
f->write_start = s;
460
if (pwrite(f->file_fh, block, size, offset) != size)
461
error_exit(f->fi_file_path, errno);
462
t = my_clock();
463
f->write_start = 0;
464
f->write_time += (t - s);
465
f->write_count += size;
466
467
/* Does this block touch the previous block? */
468
if (f->last_block_offset == -1 ||
469
offset < f->last_block_offset ||
470
offset > f->last_block_offset + f->last_block_size)
471
/* If not, it is a new block (with a gap): */
472
f->block_write_count++;
473
f->last_block_offset = offset;
474
f->last_block_size = size;
475
}
476
477
static void read_file(File *f, void *block, size_t size, off_t start)
478
{
479
my_time t, s;
480
481
s = my_clock();
482
f->read_start = s;
483
if (pread(f->file_fh, block, size, start) != size)
484
error_exit(f->fi_file_path, errno);
485
t = my_clock();
486
f->read_start = 0;
487
f->read_time += (t - s);
488
f->read_count += size;
489
}
490
491
static void sync_file(File *f)
492
{
493
my_time t, s;
494
495
s = my_clock();
496
f->flush_start = s;
497
if (fsync(f->file_fh) == -1)
498
error_exit(f->fi_file_path, errno);
499
t = my_clock();
500
f->flush_start = 0;
501
f->flush_time += t - s;
502
f->flush_count++;
503
}
504
505
static void new_file(File **ret_f, const char *test_name, const char *monitor_name)
506
{
507
File *f;
508
509
f = malloc(sizeof(File));
510
memset(f, 0, sizeof(File));
511
f->last_block_offset = (off_t) -1;
512
f->rs_min_block_offset = (off_t) -1;
513
514
monitor_file(f, test_name, monitor_name);
515
f->fi_write = write_file;
516
f->fi_sync = sync_file;
517
518
*ret_f = f;
519
}
520
521
static void open_file(File *f)
522
{
523
f->file_fh = open(f->fi_file_path, O_RDWR, 0);
524
if (f->file_fh == -1)
525
error_exit(f->fi_file_path, errno);
526
527
f->total_time = my_clock();
528
}
529
530
static void close_file(File *f)
531
{
532
f->fi_sync(f);
533
f->total_time = my_clock() - f->total_time;
534
if (f->file_fh != -1)
535
close(f->file_fh);
536
537
unmonitor_file(f);
538
539
printf("\n=* TEST: %s (%s) *=\n", f->fi_test_name, f->fi_monitor_name);
540
printf("Written K: %.2f\n", (double) f->write_count / (double) 1024);
541
printf("Run time: %.2f ms\n", (double) f->total_time / (double) 1000);
542
if (f->rs_flush_block_total > 0)
543
printf("Flush blks: %lu\n", f->rs_flush_block_total);
544
if (f->write_count > 0) {
545
printf("Tot blocks: %lu\n", f->block_write_count);
546
printf("Seek time: %.3f ms\n", (double) f->flush_time / (double) f->block_write_count / (double) 1000);
547
}
548
printf("\n");
549
if (f->write_time)
550
printf("Write K/s: %.2f\n", (double) f->write_count * (double) 1000000 / (double) 1024 / (double) f->write_time);
551
if (f->read_time)
552
printf("Read K/s: %.2f\n", (double) f->read_count * (double) 1000000 / (double) 1024 / (double) f->read_time);
553
if (f->flush_time)
554
printf("Flush K/s: %.2f\n", (double) f->write_count * (double) 1000000 / (double) 1024 / (double) f->flush_time);
555
if (f->write_time + f->read_time + f->flush_time)
556
printf("Total K/s: %.2f\n", (double) f->write_count * (double) 1000000 / (double) 1024 / ((double) f->write_time + (double) f->read_time + (double) f->flush_time));
557
printf("=*=\n");
558
}
559
560
/* ------------------- SORTED I/O ------------------- */
561
562
/* Sort records before they are writing.
563
* Options are also added to write records,
564
* in large blocks. This requires reading the block
565
* first.
566
*/
567
568
static int compare_rec(const void *a, const void *b)
569
{
570
SortedRec *ra = (SortedRec *) a;
571
SortedRec *rb = (SortedRec *) b;
572
573
if (ra->sr_offset == rb->sr_offset) {
574
if (ra->sr_order == rb->sr_order)
575
return 0;
576
if (ra->sr_order < rb->sr_order)
577
return -1;
578
return 1;
579
}
580
if (ra->sr_offset < rb->sr_offset)
581
return -1;
582
return 1;
583
}
584
585
static void sorted_write_all(File *f)
586
{
587
SortedRec *rec;
588
int i;
589
590
qsort(f->sf_records, f->sf_rec_count, sizeof(SortedRec), compare_rec);
591
rec = f->sf_records;
592
for (i=0; i<f->sf_rec_count; i++) {
593
write_file(f, rec->sr_data, SIM_RECORD_SIZE, rec->sr_offset);
594
rec++;
595
}
596
597
f->sf_rec_count = 0;
598
f->sf_alloc_pos = 0;
599
f->sf_order = 0;
600
}
601
602
static void sorted_write_rw_all(File *f)
603
{
604
SortedRec *rec;
605
int i;
606
off_t offset = 0;
607
size_t size = 0;
608
609
qsort(f->sf_records, f->sf_rec_count, sizeof(SortedRec), compare_rec);
610
rec = f->sf_records;
611
for (i=0; i<f->sf_rec_count; i++) {
612
reread:
613
if (!size) {
614
offset = (rec->sr_offset / 1024) * 1024;
615
size = SORTED_BUFFER_SIZE;
616
if (offset+(off_t)size > SIM_FILE_SIZE)
617
size = (size_t) (SIM_FILE_SIZE - offset);
618
read_file(f, f->sf_buffer, size, offset);
619
}
620
621
if (rec->sr_offset >= offset && rec->sr_offset+SIM_RECORD_SIZE <= offset+size)
622
memcpy(&f->sf_buffer[rec->sr_offset - offset], rec->sr_data, SIM_RECORD_SIZE);
623
else {
624
write_file(f, f->sf_buffer, size, offset);
625
size = 0;
626
goto reread;
627
}
628
rec++;
629
}
630
631
f->sf_rec_count = 0;
632
f->sf_alloc_pos = 0;
633
f->sf_order = 0;
634
}
635
636
static void sorted_write_rw_no_gaps_all(File *f)
637
{
638
SortedRec *rec;
639
int i;
640
off_t offset;
641
size_t size = 0;
642
643
qsort(f->sf_records, f->sf_rec_count, sizeof(SortedRec), compare_rec);
644
rec = f->sf_records;
645
if (!f->sf_rec_count)
646
goto done;
647
648
offset = (rec->sr_offset / 1024) * 1024;
649
for (i=0; i<f->sf_rec_count; i++) {
650
reread:
651
if (!size) {
652
size = SORTED_BUFFER_SIZE;
653
if (offset+size > SIM_FILE_SIZE)
654
size = SIM_FILE_SIZE - offset;
655
read_file(f, f->sf_buffer, size, offset);
656
}
657
658
if (rec->sr_offset >= offset && rec->sr_offset+SIM_RECORD_SIZE <= offset+size)
659
memcpy(&f->sf_buffer[rec->sr_offset - offset], rec->sr_data, SIM_RECORD_SIZE);
660
else {
661
write_file(f, f->sf_buffer, size, offset);
662
offset += size;
663
if (rec->sr_offset < offset)
664
offset = (rec->sr_offset / 1024) * 1024;
665
size = 0;
666
goto reread;
667
}
668
rec++;
669
}
670
671
done:
672
f->sf_rec_count = 0;
673
f->sf_alloc_pos = 0;
674
f->sf_order = 0;
675
}
676
677
static void sorted_sync_file(File *f)
678
{
679
f->fi_write_all(f);
680
sync_file(f);
681
}
682
683
static void sorted_write_file(File *f, void *block, size_t size, off_t offset)
684
{
685
SortedRec *rec;
686
687
if (size != SIM_RECORD_SIZE)
688
printf("ooops\n");
689
690
if (f->sf_rec_count == SORTED_MAX_RECORDS ||
691
f->sf_alloc_pos + size > SORTED_DATA_SIZE) {
692
f->fi_write_all(f);
693
}
694
695
rec = &f->sf_records[f->sf_rec_count];
696
rec->sr_offset = offset;
697
rec->sr_order = f->sf_order;
698
rec->sr_data = &f->sf_data[f->sf_alloc_pos];
699
memcpy(rec->sr_data, block, size);
700
f->sf_alloc_pos += size;
701
f->sf_rec_count++;
702
f->sf_order++;
703
}
704
705
/* ------------------- RE-WRITE FLUSH ------------------- */
706
707
/* The idea is that it is better to re-write the file
708
* sequentially, then allow the FS to write scattered
709
* dirty blocks!
710
*
711
* This comes from the fact that seeking is a lot more
712
* expensive than sequential write.
713
*/
714
715
static void rewrite_all_sync_file(File *f)
716
{
717
off_t offset = 0;
718
size_t size = 0;
719
720
while (offset < SIM_FILE_SIZE) {
721
size = SORTED_BUFFER_SIZE;
722
if (offset + size > SIM_FILE_SIZE)
723
size = SIM_FILE_SIZE - offset;
724
read_file(f, f->sf_buffer, size, offset);
725
write_file(f, f->sf_buffer, size, offset);
726
offset += size;
727
}
728
729
sync_file(f);
730
}
731
732
static void rewrite_min_max_sync_file(File *f)
733
{
734
off_t offset = 0;
735
size_t size = 0;
736
off_t eof = SIM_FILE_SIZE;
737
738
if (f->rs_min_block_offset != (off_t) -1)
739
offset = f->rs_min_block_offset / 1024 * 1024;
740
eof = (f->rs_max_block_offset + 1023) / 1024 * 1024;
741
if (eof > SIM_FILE_SIZE)
742
eof = SIM_FILE_SIZE;
743
while (offset < eof) {
744
size = SORTED_BUFFER_SIZE;
745
if (offset + size > eof)
746
size = eof - offset;
747
read_file(f, f->sf_buffer, size, offset);
748
write_file(f, f->sf_buffer, size, offset);
749
offset += size;
750
}
751
752
sync_file(f);
753
}
754
755
static void rewrite_min_max_write_file(File *f, void *block, size_t size, off_t offset)
756
{
757
off_t top_offset;
758
759
write_file(f, block, size, offset);
760
761
/* Round up and down by 1K */
762
top_offset = offset + size;
763
offset = offset / 1024 * 1024;
764
top_offset = (top_offset + 1023) / 1024 * 1024;
765
size = (size_t) (top_offset - offset);
766
767
/* Calculate max and min and max offset: */
768
if (f->rs_min_block_offset == (off_t) -1 || offset < f->rs_min_block_offset)
769
f->rs_min_block_offset = offset;
770
if (offset + size > f->rs_max_block_offset)
771
f->rs_max_block_offset = offset + size;
772
}
773
774
static void rewrite_opt_rewrite_file(File *f)
775
{
776
RewriteRec *rec;
777
int i;
778
off_t offset;
779
off_t size;
780
size_t tfer;
781
782
/* Re-write all areas written: */
783
rec = f->rs_records;
784
for (i=0; i<f->rs_rec_count; i++) {
785
size = rec->rr_size;
786
offset = rec->rr_offset;
787
while (size) {
788
tfer = SORTED_BUFFER_SIZE;
789
if ((off_t) tfer > size)
790
tfer = size;
791
read_file(f, f->sf_buffer, tfer, offset);
792
write_file(f, f->sf_buffer, tfer, offset);
793
offset += tfer;
794
size -= tfer;
795
}
796
rec++;
797
}
798
799
f->rs_flush_block_total += f->rs_rec_count;
800
f->rs_rec_count = 0;
801
}
802
803
static int rewrite_opt_comp(void *k, void *r)
804
{
805
register off_t *key = (off_t *) k;
806
register RewriteRec *rec = (RewriteRec *) r;
807
808
if (*key == rec->rr_offset)
809
return 0;
810
if (*key < rec->rr_offset)
811
return -1;
812
return 1;
813
}
814
815
static void rewrite_opt_sync_file(File *f)
816
{
817
rewrite_opt_rewrite_file(f);
818
sync_file(f);
819
}
820
821
static void rewrite_opt_write_file(File *f, void *block, size_t size, off_t offset)
822
{
823
RewriteRec *rec;
824
size_t idx;
825
off_t top_offset;
826
827
write_file(f, block, size, offset);
828
829
/* Round up and down by 1K */
830
top_offset = offset + size;
831
offset = offset / 1024 * 1024;
832
top_offset = (top_offset + 1023) / 1024 * 1024;
833
size = (size_t) (top_offset - offset);
834
835
if ((rec = my_bsearch(&offset, f->rs_records, f->rs_rec_count, sizeof(RewriteRec), &idx, rewrite_opt_comp))) {
836
if ((off_t) size > rec->rr_size)
837
rec->rr_size = size;
838
goto merge_right;
839
}
840
841
if (idx == 0) {
842
/* The offset is before the first entry. */
843
if (idx < f->rs_rec_count) {
844
/* There is a first entry: */
845
rec = f->rs_records;
846
if (rec->rr_offset - (offset + size) < REWRITE_BLOCK_DISTANCE)
847
goto add_to_right;
848
}
849
850
/* Add the first entry: */
851
goto add_the_entry;
852
}
853
854
/* Not the first entry: */
855
idx--;
856
rec = f->rs_records + idx;
857
858
if (offset - (rec->rr_offset + rec->rr_size) < REWRITE_BLOCK_DISTANCE) {
859
/* Add to block on left: */
860
size = (offset + size) - rec->rr_offset;
861
if (size > rec->rr_size)
862
rec->rr_size = size;
863
goto merge_right;
864
}
865
866
idx++;
867
rec = f->rs_records + idx;
868
869
if (idx < f->rs_rec_count && rec->rr_offset - (offset + size) < REWRITE_BLOCK_DISTANCE)
870
goto add_to_right;
871
872
add_the_entry:
873
if (f->rs_rec_count == REWRITE_MAX_RECORDS) {
874
rewrite_opt_rewrite_file(f);
875
idx = 0;
876
}
877
rec = f->rs_records + idx;
878
memmove(rec+1, rec, (f->rs_rec_count - idx) * sizeof(RewriteRec));
879
rec->rr_offset = offset;
880
rec->rr_size = (off_t) size;
881
f->rs_rec_count++;
882
return;
883
884
add_to_right:
885
rec->rr_size += rec->rr_offset - offset;
886
if (size > rec->rr_size)
887
rec->rr_size = size;
888
rec->rr_offset = offset;
889
890
merge_right:
891
if (idx+1 < f->rs_rec_count) {
892
/* There is a record right: */
893
if (rec->rr_offset + rec->rr_size + REWRITE_BLOCK_DISTANCE > (rec+1)->rr_offset) {
894
/* Merge and remove! */
895
size = (rec+1)->rr_size + ((rec+1)->rr_offset - rec->rr_offset);
896
if (size > rec->rr_size)
897
rec->rr_size = size;
898
f->rs_rec_count--;
899
assert(f->rs_rec_count > idx);
900
memmove(rec+1, rec+2, (f->rs_rec_count - idx - 1) * sizeof(RewriteRec));
901
}
902
}
903
return;
904
905
}
906
907
static void rewrite_limit_sync_file(File *f)
908
{
909
rewrite_opt_rewrite_file(f);
910
sync_file(f);
911
}
912
913
/*
914
* This options is like opt but it limits the number of
915
* blocks that can be written.
916
*/
917
static void rewrite_limit_write_file(File *f, void *block, size_t size, off_t offset)
918
{
919
RewriteRec *rec;
920
921
/* There must always be room for one more: */
922
assert(f->rs_rec_count < REWRITE_RECORD_LIMIT);
923
rewrite_opt_write_file(f, block, size, offset);
924
if (f->rs_rec_count == REWRITE_RECORD_LIMIT) {
925
/* Consolidate 2 blocks that are closest to each other in other to
926
* make space for another block:
927
*/
928
int i, idx;
929
off_t gap;
930
off_t min_gap = (off_t) -1;
931
932
rec = f->rs_records;
933
for (i=0; i<f->rs_rec_count-1; i++) {
934
gap = (rec+1)->rr_offset - (rec->rr_offset + rec->rr_size);
935
if (min_gap == (off_t) -1 || gap < min_gap) {
936
idx = i;
937
min_gap = gap;
938
}
939
rec++;
940
}
941
942
/* Merge this with the next: */
943
rec = f->rs_records + idx;
944
size = (rec+1)->rr_size + ((rec+1)->rr_offset - rec->rr_offset);
945
if (size > rec->rr_size)
946
rec->rr_size = size;
947
f->rs_rec_count--;
948
assert(f->rs_rec_count > idx);
949
memmove(rec+1, rec+2, (f->rs_rec_count - idx - 1) * sizeof(RewriteRec));
950
}
951
}
952
953
/* ------------------- SIMULATION I/O ------------------- */
954
955
/*
956
static void random_read_bytes(File *f, off_t file_size, size_t size, size_t count, int print)
957
{
958
char *block;
959
size_t i;
960
off_t offset;
961
long x = (long) (file_size / (off_t) size), y;
962
963
block = (char *) malloc(size);
964
965
for (i=0; i<count; i++) {
966
y = random() % x;
967
offset = (off_t) y * (off_t) size;
968
if (offset+size > file_size) {
969
printf("NOOO\n");
970
exit(1);
971
}
972
read_file(f, block, size, offset);
973
if (print) {
974
if ((i%100) == 0)
975
printf("%ld\n", (long) i);
976
}
977
}
978
free(block);
979
}
980
*/
981
982
static void read_write_bytes(File *f, off_t file_size)
983
{
984
off_t offset = 0;
985
size_t tfer;
986
987
while (file_size > 0) {
988
tfer = SORTED_BUFFER_SIZE;
989
if ((off_t) tfer > file_size)
990
tfer = (size_t) file_size;
991
read_file(f, f->sf_buffer, tfer, offset);
992
f->fi_write(f, f->sf_buffer, tfer, offset);
993
offset += tfer;
994
file_size -= tfer;
995
}
996
}
997
998
static void random_write_bytes(File *f, off_t file_size, size_t size, off_t amount_to_write, int sync_after, int pause_after)
999
{
1000
char *block;
1001
off_t offset;
1002
long x = (long) (file_size / (off_t) size), y;
1003
long pbytes_written = 0;
1004
long sbytes_written = 0;
1005
off_t total_written = 0;
1006
int i;
1007
1008
block = (char *) malloc(size);
1009
for (i=0; i<size; i++)
1010
block[i] = (char) i;
1011
1012
if (pause_after) {
1013
pbytes_written = 0;
1014
sleep(10);
1015
}
1016
while (total_written < amount_to_write) {
1017
y = random() % x;
1018
do {
1019
offset = (off_t) y * (off_t) size;
1020
}
1021
while (offset+size > file_size);
1022
f->fi_write(f, block, size, offset);
1023
sbytes_written += size;
1024
pbytes_written += size;
1025
total_written += size;
1026
if (sync_after && sbytes_written > sync_after) {
1027
sbytes_written = 0;
1028
f->fi_sync(f);
1029
}
1030
if (pause_after && pbytes_written > pause_after) {
1031
pbytes_written = 0;
1032
sleep(10);
1033
}
1034
}
1035
free(block);
1036
}
1037
1038
static void seq_write_bytes(File *f, off_t file_size, size_t size, off_t amount_to_write, int sync_after, int pause_after)
1039
{
1040
char *block;
1041
off_t offset;
1042
long pbytes_written = 0;
1043
long sbytes_written = 0;
1044
off_t total_written = 0;
1045
int i;
1046
1047
block = (char *) malloc(size);
1048
for (i=0; i<size; i++)
1049
block[i] = (char) i;
1050
1051
if (pause_after) {
1052
pbytes_written = 0;
1053
sleep(10);
1054
}
1055
offset = 0;
1056
while (total_written < amount_to_write) {
1057
if (offset+size > file_size)
1058
offset = 0;
1059
f->fi_write(f, block, size, offset);
1060
sbytes_written += size;
1061
pbytes_written += size;
1062
total_written += size;
1063
offset += size;
1064
if (sync_after && sbytes_written > sync_after) {
1065
sbytes_written = 0;
1066
f->fi_sync(f);
1067
}
1068
if (pause_after && pbytes_written > pause_after) {
1069
pbytes_written = 0;
1070
sleep(10);
1071
}
1072
}
1073
free(block);
1074
}
1075
1076
static void simulate_xlog_write(File *f, off_t eof, off_t start_point, size_t size, off_t amount_to_write)
1077
{
1078
char *block;
1079
off_t start = start_point;
1080
off_t offset;
1081
size_t tfer;
1082
off_t total_written = 0;
1083
int i;
1084
1085
block = (char *) malloc(1024*16);
1086
for (i=0; i<1024*16; i++)
1087
block[i] = (char) i;
1088
1089
while (total_written < amount_to_write) {
1090
/* Write 512 byte boundary block around random data we wish to write. */
1091
offset = (start / 512) * 512;
1092
tfer = (((start + size) / 512) * 512 + 512) - offset;
1093
f->fi_write(f, block, tfer, offset);
1094
total_written += tfer;
1095
f->fi_sync(f);
1096
start += size;
1097
if (start + size > eof)
1098
start = start_point;
1099
}
1100
1101
free(block);
1102
}
1103
1104
static void seq_write_blocks_aligned(File *f, off_t file_size, size_t block_size, off_t amount_to_write)
1105
{
1106
char *block;
1107
off_t i, no_of_writes;
1108
off_t offset, inc;
1109
1110
/* Blocks, 1 K in size: */
1111
block_size = block_size / 1024 * 1024;
1112
1113
block = (char *) malloc(block_size);
1114
for (i=0; i<block_size; i++)
1115
block[i] = (char) i;
1116
1117
no_of_writes = amount_to_write / block_size;
1118
1119
/* Calculate and round increment: */
1120
inc = (file_size / no_of_writes) / 1024 * 1024;
1121
1122
/* This is required if we want a gap: */
1123
assert(inc > block_size);
1124
1125
offset = 0;
1126
for (i=0; i<no_of_writes; i++) {
1127
f->fi_write(f, block, block_size, offset);
1128
offset += inc;
1129
}
1130
}
1131
1132
/* ------------------- TESTS ------------------- */
1133
1134
static void *test_xlog_write_prealloc(void *data)
1135
{
1136
File *f;
1137
1138
new_file(&f, __FUNCTION__, "xlog");
1139
create_file(f, SIM_FILE_SIZE, PREFILL);
1140
open_file(f);
1141
simulate_xlog_write(f, SIM_FILE_SIZE, 0, SIM_RECORD_SIZE, SIM_WRITE_AMOUNT);
1142
close_file(f);
1143
return NULL;
1144
}
1145
1146
static void *test_xlog_write_no_prealloc(void *data)
1147
{
1148
File *f;
1149
1150
new_file(&f, __FUNCTION__, "xlog");
1151
create_file(f, 512, TRUNCATE);
1152
open_file(f);
1153
simulate_xlog_write(f, SIM_FILE_SIZE, 0, SIM_RECORD_SIZE, SIM_WRITE_AMOUNT);
1154
close_file(f);
1155
return NULL;
1156
}
1157
1158
static void *test_xlog_write_set_eof(void *data)
1159
{
1160
File *f;
1161
1162
new_file(&f, __FUNCTION__, "xlog");
1163
create_file(f, SIM_FILE_SIZE, SET_EOF);
1164
open_file(f);
1165
simulate_xlog_write(f, SIM_FILE_SIZE, 0, SIM_RECORD_SIZE, SIM_WRITE_AMOUNT);
1166
close_file(f);
1167
return NULL;
1168
}
1169
1170
static void *test_rnd_write_prealloc(void *data)
1171
{
1172
File *f;
1173
1174
new_file(&f, __FUNCTION__, "rndw");
1175
create_file(f, SIM_FILE_SIZE, PREFILL);
1176
open_file(f);
1177
random_write_bytes(f, SIM_FILE_SIZE, SIM_RECORD_SIZE, SIM_WRITE_AMOUNT, SIM_FLUSH_THRESHOLD, SIM_PAUSE_THRESHOLD);
1178
close_file(f);
1179
return NULL;
1180
}
1181
1182
static void *test_rnd_write_no_prealloc(void *data)
1183
{
1184
File *f;
1185
1186
new_file(&f, __FUNCTION__, "rndw");
1187
create_file(f, 0, TRUNCATE);
1188
open_file(f);
1189
random_write_bytes(f, SIM_FILE_SIZE, SIM_RECORD_SIZE, SIM_WRITE_AMOUNT, SIM_FLUSH_THRESHOLD, SIM_PAUSE_THRESHOLD);
1190
close_file(f);
1191
return NULL;
1192
}
1193
1194
static void *test_rnd_write_rewrite_all_sync_prealloc(void *data)
1195
{
1196
File *f;
1197
1198
new_file(&f, __FUNCTION__, "rndw");
1199
create_file(f, SIM_FILE_SIZE, PREFILL);
1200
open_file(f);
1201
f->fi_sync = rewrite_all_sync_file;
1202
random_write_bytes(f, SIM_FILE_SIZE, SIM_RECORD_SIZE, SIM_WRITE_AMOUNT, SIM_FLUSH_THRESHOLD, SIM_PAUSE_THRESHOLD);
1203
close_file(f);
1204
return NULL;
1205
}
1206
1207
static void *test_rnd_write_rewrite_min_max_sync_prealloc(void *data)
1208
{
1209
File *f;
1210
1211
new_file(&f, __FUNCTION__, "rndw");
1212
create_file(f, SIM_FILE_SIZE, PREFILL);
1213
open_file(f);
1214
f->fi_write = rewrite_min_max_write_file;
1215
f->fi_sync = rewrite_min_max_sync_file;
1216
random_write_bytes(f, SIM_FILE_SIZE, SIM_RECORD_SIZE, SIM_WRITE_AMOUNT, SIM_FLUSH_THRESHOLD, SIM_PAUSE_THRESHOLD);
1217
close_file(f);
1218
return NULL;
1219
}
1220
1221
static void *test_rnd_write_rewrite_opt_sync_prealloc(void *data)
1222
{
1223
File *f;
1224
1225
new_file(&f, __FUNCTION__, "rndw");
1226
create_file(f, SIM_FILE_SIZE, PREFILL);
1227
open_file(f);
1228
f->fi_write = rewrite_opt_write_file;
1229
f->fi_sync = rewrite_opt_sync_file;
1230
random_write_bytes(f, SIM_FILE_SIZE, SIM_RECORD_SIZE, SIM_WRITE_AMOUNT, SIM_FLUSH_THRESHOLD, SIM_PAUSE_THRESHOLD);
1231
close_file(f);
1232
return NULL;
1233
}
1234
1235
static void *test_rnd_write_rewrite_limit_sync_prealloc(void *data)
1236
{
1237
File *f;
1238
1239
new_file(&f, __FUNCTION__, "rndw");
1240
create_file(f, SIM_FILE_SIZE, PREFILL);
1241
open_file(f);
1242
f->fi_write = rewrite_limit_write_file;
1243
f->fi_sync = rewrite_limit_sync_file;
1244
random_write_bytes(f, SIM_FILE_SIZE, SIM_RECORD_SIZE, SIM_WRITE_AMOUNT, SIM_FLUSH_THRESHOLD, SIM_PAUSE_THRESHOLD);
1245
close_file(f);
1246
return NULL;
1247
}
1248
1249
static void *test_rnd_write_sorted_prealloc(void *data)
1250
{
1251
File *f;
1252
1253
new_file(&f, __FUNCTION__, "rndsw");
1254
create_file(f, SIM_FILE_SIZE, PREFILL);
1255
open_file(f);
1256
f->fi_write = sorted_write_file;
1257
f->fi_sync = sorted_sync_file;
1258
f->fi_write_all = sorted_write_all;
1259
f->fi_write_all = sorted_write_rw_all;
1260
f->fi_write_all = sorted_write_rw_no_gaps_all;
1261
random_write_bytes(f, SIM_FILE_SIZE, SIM_RECORD_SIZE, SIM_WRITE_AMOUNT, SIM_FLUSH_THRESHOLD, SIM_PAUSE_THRESHOLD);
1262
close_file(f);
1263
return NULL;
1264
}
1265
1266
static void *test_seq_write_prealloc(void *data)
1267
{
1268
File *f;
1269
1270
new_file(&f, __FUNCTION__, "seqw");
1271
create_file(f, SIM_FILE_SIZE, PREFILL);
1272
open_file(f);
1273
seq_write_bytes(f, SIM_FILE_SIZE, SIM_RECORD_SIZE, SIM_WRITE_AMOUNT, SIM_FLUSH_THRESHOLD, SIM_PAUSE_THRESHOLD);
1274
close_file(f);
1275
return NULL;
1276
}
1277
1278
static void *test_seq_write_no_prealloc(void *data)
1279
{
1280
File *f;
1281
1282
new_file(&f, __FUNCTION__, "seqw");
1283
create_file(f, 0, TRUNCATE);
1284
open_file(f);
1285
seq_write_bytes(f, SIM_FILE_SIZE, SIM_RECORD_SIZE, SIM_WRITE_AMOUNT, SIM_FLUSH_THRESHOLD, SIM_PAUSE_THRESHOLD);
1286
close_file(f);
1287
return NULL;
1288
}
1289
1290
static void *test_seq_rw_prealloc(void *data)
1291
{
1292
File *f;
1293
1294
new_file(&f, __FUNCTION__, "seqrw");
1295
create_file(f, SIM_FILE_SIZE, PREFILL);
1296
open_file(f);
1297
read_write_bytes(f, SIM_FILE_SIZE);
1298
close_file(f);
1299
return NULL;
1300
}
1301
1302
int block_write_block_count = 128;
1303
1304
static void *test_block_write_aligned_prealloc(void *data)
1305
{
1306
File *f;
1307
off_t write_amount = SIM_FILE_SIZE / 2;
1308
1309
new_file(&f, __FUNCTION__, "block");
1310
create_file(f, SIM_FILE_SIZE, PREFILL);
1311
open_file(f);
1312
seq_write_blocks_aligned(f, SIM_FILE_SIZE, write_amount / block_write_block_count, write_amount);
1313
close_file(f);
1314
return NULL;
1315
}
1316
1317
/* ------------------- THREADING ------------------- */
1318
1319
static pthread_t run_task_as_thread(void *(*task)(void *))
1320
{
1321
pthread_t thread;
1322
int err;
1323
1324
err = pthread_create(&thread, NULL, task, NULL);
1325
if (err)
1326
error_exit("pthread_create", err);
1327
return thread;
1328
}
1329
1330
static void wait_for_task(pthread_t thread)
1331
{
1332
void *value;
1333
1334
pthread_join(thread, &value);
1335
}
1336
1337
/* ------------------- MAIN ------------------- */
1338
1339
static void reference_tests()
1340
{
1341
test_xlog_write_prealloc(NULL);
1342
test_xlog_write_no_prealloc(NULL);
1343
test_xlog_write_set_eof(NULL);
1344
1345
test_rnd_write_prealloc(NULL);
1346
test_rnd_write_no_prealloc(NULL);
1347
test_rnd_write_rewrite_all_sync_prealloc(NULL);
1348
test_rnd_write_rewrite_min_max_sync_prealloc(NULL);
1349
test_rnd_write_rewrite_opt_sync_prealloc(NULL);
1350
test_rnd_write_rewrite_limit_sync_prealloc(NULL);
1351
1352
test_rnd_write_sorted_prealloc(NULL);
1353
1354
test_seq_write_prealloc(NULL);
1355
test_seq_write_no_prealloc(NULL);
1356
test_seq_rw_prealloc(NULL);
1357
1358
test_block_write_aligned_prealloc(NULL);
1359
}
1360
1361
static void do_task(void *(*task)(void *))
1362
{
1363
pthread_t t1;
1364
t1 = run_task_as_thread(task);
1365
wait_for_task(t1);
1366
}
1367
1368
static void do_test_1()
1369
{
1370
pthread_t t1;
1371
//pthread_t t2;
1372
1373
t1 = run_task_as_thread(test_rnd_write_sorted_prealloc);
1374
//t2 = run_task_as_thread(test_xlog_write_prealloc);
1375
1376
wait_for_task(t1);
1377
//wait_for_task(t2);
1378
}
1379
1380
static void do_test_2()
1381
{
1382
//do_task(test_rnd_write_prealloc);
1383
//do_task(test_rnd_write_rewrite_min_max_sync_prealloc);
1384
do_task(test_rnd_write_rewrite_limit_sync_prealloc);
1385
1386
/*
1387
block_write_block_count = 1;
1388
do_task(test_block_write_aligned_prealloc);
1389
block_write_block_count = 2;
1390
do_task(test_block_write_aligned_prealloc);
1391
block_write_block_count = 4;
1392
do_task(test_block_write_aligned_prealloc);
1393
block_write_block_count = 8;
1394
do_task(test_block_write_aligned_prealloc);
1395
block_write_block_count = 16;
1396
do_task(test_block_write_aligned_prealloc);
1397
block_write_block_count = 32;
1398
do_task(test_block_write_aligned_prealloc);
1399
block_write_block_count = 64;
1400
do_task(test_block_write_aligned_prealloc);
1401
block_write_block_count = 128;
1402
do_task(test_block_write_aligned_prealloc);
1403
*/
1404
}
1405
1406
int main(int argc, char **argv)
1407
{
1408
//pthread_t m;
1409
1410
//m = run_task_as_thread(iotest_monitor);
1411
1412
do_test_2();
1413
1414
monitor_running = FALSE;
1415
//wait_for_task(m);
1416
return 0;
1417
}
Loggerhead is a web-based interface for Breezy
Version: 2.0.1