~drizzle-trunk/drizzle/development

void MSRepoFile::sendBlob(MSOpenTable *otab, uint64_t offset, uint64_t req_offset, uint64_t req_size, uint32_t auth_code, bool with_auth_code, bool info_only, CSHTTPOutputStream *stream)

186

{

187

MSConnectionHandler *me;

188

size_t tfer;

189

off_t start_offset = offset;

190

MSBlobHeadRec blob_head;

191

uint8_t storage_type;

192

uint16_t head_size, meta_size;

193

uint64_t blob_data_size, local_blob_size, meta_offset;

194

uint32_t ac;

195

char num_str[CS_WIDTH_INT_64];

196

bool redirect = false;

197

198

199

enter_();

200

me = (MSConnectionHandler *) self;

201

202

read(&blob_head, start_offset, sizeof(MSBlobHeadRec), sizeof(MSBlobHeadRec));

203

local_blob_size = CS_GET_DISK_6(blob_head.rb_blob_repo_size_6); // This is the size of the BLOB data in the repository. Can be 0 if the BLOB is stored some where else.

204

blob_data_size = CS_GET_DISK_6(blob_head.rb_blob_data_size_6);// This is the actual size of the BLOB.

205

head_size = CS_GET_DISK_2(blob_head.rb_head_size_2);

206

meta_size = CS_GET_DISK_2(blob_head.rb_mdata_size_2);

207

meta_offset = start_offset + CS_GET_DISK_2(blob_head.rb_mdata_offset_2);

208

ac = CS_GET_DISK_4(blob_head.rb_auth_code_4);

209

if ((with_auth_code && auth_code != ac) || (CS_GET_DISK_4(blob_head.rd_magic_4) != MS_BLOB_HEADER_MAGIC))

210

CSException::throwException(CS_CONTEXT, MS_ERR_AUTH_FAILED, "Invalid BLOB identifier");

211

212

storage_type = CS_GET_DISK_1(blob_head.rb_storage_type_1);

213

214

if ((!info_only) && BLOB_IN_CLOUD(storage_type)) {

215

CSString *redirect_url = NULL;

216

CloudKeyRec key;

217

getBlobKey(&blob_head, &key);

218

redirect_url = otab->getDB()->myBlobCloud->cl_getDataURL(&key);

219

push_(redirect_url);

220

stream->setStatus(301);

221

stream->addHeader("Location", redirect_url->getCString());

222

release_(redirect_url);

223

redirect = true;

224

} else

225

stream->setStatus(200);

226

227

if (storage_type == MS_STANDARD_STORAGE) {

228

char hex_checksum[33];

229

cs_bin_to_hex(33, hex_checksum, 16, blob_head.rb_blob_checksum_md5d.val);

230

hex_checksum[32] = 0;

231

stream->addHeader(MS_CHECKSUM_TAG, hex_checksum);

232

}

233

234

snprintf(num_str, CS_WIDTH_INT_64, "%"PRIu64"", blob_data_size);

235

stream->addHeader(MS_BLOB_SIZE, num_str);

236

237

snprintf(num_str, CS_WIDTH_INT_64, "%"PRIu32"", CS_GET_DISK_4(blob_head.rb_last_access_4));

238

stream->addHeader(MS_LAST_ACCESS, num_str);

239

240

snprintf(num_str, CS_WIDTH_INT_64, "%"PRIu32"", CS_GET_DISK_4(blob_head.rb_access_count_4));

241

stream->addHeader(MS_ACCESS_COUNT, num_str);

242

243

snprintf(num_str, CS_WIDTH_INT_64, "%"PRIu32"", CS_GET_DISK_4(blob_head.rb_create_time_4));

244

stream->addHeader(MS_CREATION_TIME, num_str);

245

246

snprintf(num_str, CS_WIDTH_INT_64, "%"PRIu32"", storage_type);

247

stream->addHeader(MS_BLOB_TYPE, num_str);

248

249

250

// Add the meta data headers.

251

if (meta_size) {

252

MetaData metadata;

253

char *name, *value;

254

255

read(otab->myOTBuffer, meta_offset, meta_size, meta_size);

256

metadata.use_data(otab->myOTBuffer, meta_size);

257

while ((name = metadata.findNext(&value))) {

258

stream->addHeader(name, value);

259

}

260

261

}

262

263

offset += (uint64_t) head_size + req_offset;

264

local_blob_size -= req_offset;

265

if (local_blob_size > req_size)

266

local_blob_size = req_size;

267

268

stream->setContentLength((redirect || info_only)?0:local_blob_size);

269

stream->writeHead();

270

me->replyPending = false;

271

272

if ((!redirect) && !info_only) {

273

274

while (local_blob_size > 0) {

275

if (local_blob_size <= MS_OT_BUFFER_SIZE)

276

tfer = (size_t) local_blob_size;

277

else

278

tfer = MS_OT_BUFFER_SIZE;

279

read(otab->myOTBuffer, offset, tfer, tfer);

280

stream->write(otab->myOTBuffer, tfer);

281

offset += (uint64_t) tfer;

282

local_blob_size -= (uint64_t) tfer;

283

}

284

}

285

stream->flush();

286

287

if (!info_only) {

288

// Should the time stamp be updated if only the BLOB info was requested?

289

/* Update the access timestamp: */

290

updateAccess(&blob_head, start_offset);

291

}

292

293

exit_();

294

}

295

296

void MSRepoFile::update_blob_header(MSOpenTable *otab, uint64_t offset, uint64_t blob_size, uint16_t head_size, uint16_t new_head_size)

297

{

298

uint16_t w_offset = offsetof(MSBlobHeadRec, rb_ref_count_2);

299

MSRepoPointersRec ptr;

300

enter_();

301

302

ptr.rp_chars = otab->myOTBuffer;

303

CS_SET_DISK_4(ptr.rp_head->rb_mod_time_4, time(NULL));

304

305

if (head_size == new_head_size) {

306

w_offset = offsetof(MSBlobHeadRec, rb_ref_count_2);

307

write(otab->myOTBuffer + w_offset, offset + w_offset, head_size - w_offset);

308

} else {

309

/* Copy to a new space, free the old: */

310

off_t dst_offset;

311

CSStringBuffer *buffer;

312

MSRepoPointersRec ptr;

313

uint16_t ref_count, ref_size;

314

uint32_t tab_id;

315

uint64_t blob_id;

316

317

myRepo->myRepoDatabase->openWriteRepo(otab);

318

dst_offset = otab->myWriteRepo->myRepoFileSize;

319

320

/* Write the header. */

321

otab->myWriteRepoFile->write(otab->myOTBuffer, dst_offset, new_head_size);

322

323

/* We have an engine reference, copy the BLOB over: */

324

new_(buffer, CSStringBuffer());

325

push_(buffer);

326

buffer->setLength(MS_COMPACTOR_BUFFER_SIZE);

327

CSFile::transfer(otab->myWriteRepoFile, dst_offset + new_head_size, this, offset + head_size, blob_size, buffer->getBuffer(0), MS_COMPACTOR_BUFFER_SIZE);

328

release_(buffer);

329

330

ptr.rp_chars = otab->myOTBuffer;

331

#ifdef HAVE_ALIAS_SUPPORT

332

/* Update the BLOB alias if required. */

333

334

if (CS_GET_DISK_2(ptr.rp_head->rb_alias_offset_2)) {

335

uint32_t alias_hash = CS_GET_DISK_4(ptr.rp_head->rb_alias_hash_4);

336

myRepo->myRepoDatabase->moveBlobAlias(myRepo->myRepoID, offset, alias_hash, myRepo->myRepoID, dst_offset);

337

}

338

#endif

339

340

/* Update the references: */

341

ref_size = CS_GET_DISK_1(ptr.rp_head->rb_ref_size_1);

342

ref_count = CS_GET_DISK_2(ptr.rp_head->rb_ref_count_2);

343

ptr.rp_chars += myRepo->myRepoBlobHeadSize;

344

345

while (ref_count) {

346

switch (CS_GET_DISK_2(ptr.rp_ref->rr_type_2)) {

347

case MS_BLOB_FREE_REF:

348

break;

349

case MS_BLOB_TABLE_REF:

350

tab_id = CS_GET_DISK_4(ptr.rp_tab_ref->tr_table_id_4);

351

blob_id = CS_GET_DISK_6(ptr.rp_tab_ref->tr_blob_id_6);

352

353

if (otab->getDBTable()->myTableID == tab_id)

354

otab->getDBTable()->updateBlobHandle(otab, blob_id, otab->myWriteRepo->myRepoID, dst_offset, new_head_size);

355

else {

356

MSOpenTable *ref_otab;

357

358

ref_otab = MSTableList::getOpenTableByID(myRepo->myRepoDatabase->myDatabaseID, tab_id);

359

frompool_(ref_otab);

360

ref_otab->getDBTable()->updateBlobHandle(ref_otab, blob_id, otab->myWriteRepo->myRepoID, dst_offset, new_head_size);

361

backtopool_(ref_otab);

362

}

363

break;

364

case MS_BLOB_DELETE_REF:

365

break;

366

default:

367

break;

368

}

369

ptr.rp_chars += ref_size;

370

ref_count--;

371

}

372

373

otab->myWriteRepo->myRepoFileSize += new_head_size + blob_size;

374

375

/* Free the old head: */

376

ptr.rp_chars = otab->myOTBuffer;

377

if (myRepo->lockedForBackup()) {

378

// This is done to tell the backup process that this BLOB was moved

379

// after the backup had started and needs to be backed up also.

380

// (The moved BLOB doesn't though because the change took place after the backup had begone.)

381

CS_SET_DISK_1(ptr.rp_head->rb_status_1, MS_BLOB_MOVED);

382

CS_SET_DISK_4(ptr.rp_head->rb_backup_id_4, myRepo->myRepoDatabase->backupID());

383

} else

384

CS_SET_DISK_1(ptr.rp_head->rb_status_1, MS_BLOB_DELETED);

385

386

write(ptr.rp_chars + MS_BLOB_STAT_OFFS, offset + MS_BLOB_STAT_OFFS, head_size - MS_BLOB_STAT_OFFS);

387

388

#ifdef DO_NOT_WIPE_BLOB

389

// Why is the BLOB header data being wiped here?

390

// The data may be needed for backup.

391

ptr.rp_chars += myRepo->myRepoBlobHeadSize;

392

memset(ptr.rp_chars, 0, head_size - myRepo->myRepoBlobHeadSize);

393

394

w_offset = offsetof(MSBlobHeadRec, rb_alias_hash_4);

395

write(otab->myOTBuffer + w_offset, offset + w_offset, head_size - w_offset);

396

#endif

397

398

/* Increment the garbage count: */

399

updateGarbage(head_size + blob_size);

400

401

}

402

exit_();

403

}

404

405

void MSRepoFile::referenceBlob(MSOpenTable *otab, uint64_t offset, uint16_t head_size, uint32_t tab_id, uint64_t blob_id, uint64_t blob_ref_id, uint32_t auth_code, uint16_t col_index)

406

{

407

CSMutex *lock;

408

MSRepoPointersRec ptr;

409

u_int size, ref_count;

410

size_t ref_size, read_size;

411

MSRepoBlobRefPtr free_ref = NULL;

412

MSRepoBlobRefPtr free2_ref = NULL;

413

MSRepoTableRefPtr tab_ref = NULL;

414

uint16_t new_head_size;

415

#ifdef HAVE_ALIAS_SUPPORT

416

bool reset_alias_index = false;

417

char blob_alias[BLOB_ALIAS_LENGTH];

418

#endif

419

uint64_t blob_size;

420

421

enter_();

422

/* Lock the BLOB: */

423

lock = &myRepo->myRepoLock[offset % CS_REPO_REC_LOCK_COUNT];

424

lock_(lock);

425

/* Read the header: */

426

if (head_size > MS_OT_BUFFER_SIZE) {

427

CSException::throwAssertion(CS_CONTEXT, "BLOB header overflow");

428

}

429

430

read_size = read(otab->myOTBuffer, offset, head_size, 0);

431

ptr.rp_chars = otab->myOTBuffer;

432

if (CS_GET_DISK_4(ptr.rp_head->rd_magic_4) != MS_BLOB_HEADER_MAGIC)

433

CSException::throwException(CS_CONTEXT, MS_ERR_NOT_FOUND, "Invalid BLOB identifier");

434

if (read_size < myRepo->myRepoBlobHeadSize)

435

CSException::throwException(CS_CONTEXT, MS_ERR_NOT_FOUND, "BLOB header incomplete");

436

if ( ! IN_USE_BLOB_STATUS(CS_GET_DISK_1(ptr.rp_head->rb_status_1)))

437

CSException::throwException(CS_CONTEXT, MS_ERR_NOT_FOUND, "BLOB has already been deleted");

438

if (CS_GET_DISK_4(ptr.rp_bytes + myRepo->myRepoBlobHeadSize - 4) != auth_code)

439

CSException::throwException(CS_CONTEXT, MS_ERR_NOT_FOUND, "BLOB data does not match reference");

440

/* Assume that what is here is correct: */

441

if (head_size != CS_GET_DISK_2(ptr.rp_head->rb_head_size_2)) {

442

head_size = CS_GET_DISK_2(ptr.rp_head->rb_head_size_2);

443

if (head_size > MS_OT_BUFFER_SIZE) { // Could happen if the header was creatd with a different version of PBMS.

444

CSException::throwAssertion(CS_CONTEXT, "BLOB header overflow");

445

}

446

read_size = read(otab->myOTBuffer, offset, head_size, myRepo->myRepoBlobHeadSize);

447

}

448

head_size = CS_GET_DISK_2(ptr.rp_head->rb_head_size_2);

449

blob_size = CS_GET_DISK_6(ptr.rp_head->rb_blob_repo_size_6);

450

if (read_size < head_size) {

451

/* This should not happen, because the file has been recovered,

452

* which should have already adjusted the head and blob

453

* size.

454

* If this happens then the file must have been truncated an the BLOB has been

455

* lost so we set the blob size to zero.

456

457

head_size = read_size;

458

blob_size = 0;

459

460

}

461

ref_size = CS_GET_DISK_1(ptr.rp_head->rb_ref_size_1);

462

ref_count = CS_GET_DISK_2(ptr.rp_head->rb_ref_count_2);

463

464

#ifdef HAVE_ALIAS_SUPPORT

465

if (CS_GET_DISK_2(ptr.rp_head->rb_alias_offset_2)) {

466

reset_alias_index = true;

467

strcpy(blob_alias, otab->myOTBuffer + CS_GET_DISK_2(ptr.rp_head->rb_alias_offset_2));

468

}

469

#endif

470

471

size = head_size - myRepo->myRepoBlobHeadSize;

472

if (size > ref_size * ref_count)

473

size = ref_size * ref_count;

474

CS_SET_DISK_4(ptr.rp_head->rb_last_ref_4, (uint32_t) time(NULL)); // Set the reference time

475

CS_SET_DISK_1(ptr.rp_head->rb_status_1, MS_BLOB_REFERENCED);

476

ptr.rp_chars += myRepo->myRepoBlobHeadSize;

477

while (size >= ref_size) {

478

switch (CS_GET_DISK_2(ptr.rp_ref->rr_type_2)) {

479

case MS_BLOB_FREE_REF:

480

if (!free_ref)

481

free_ref = ptr.rp_blob_ref;

482

else if (!free2_ref)

483

free2_ref = ptr.rp_blob_ref;

484

break;

485

case MS_BLOB_TABLE_REF:

486

#ifdef HAVE_ALIAS_SUPPORT

487

reset_alias_index = false; // No need to reset the index if the BLOB is already referenced. (We don't care what table references it.)

488

#endif

489

if (CS_GET_DISK_4(ptr.rp_tab_ref->tr_table_id_4) == tab_id &&

490

CS_GET_DISK_6(ptr.rp_tab_ref->tr_blob_id_6) == blob_id)

491

tab_ref = ptr.rp_tab_ref;

492

break;

493

case MS_BLOB_DELETE_REF: {

494

u_int tab_index;

495

496

tab_index = CS_GET_DISK_2(ptr.rp_temp_ref->tp_del_ref_2);

497

if (tab_index && tab_index < ref_count) {

498

MSRepoTableRefPtr tr;

499

500

tab_index--;

501

tr = (MSRepoTableRefPtr) (otab->myOTBuffer + myRepo->getRepoBlobHeadSize() + tab_index * ref_size);

502

if (CS_GET_DISK_4(tr->tr_table_id_4) == tab_id &&

503

CS_GET_DISK_6(tr->tr_blob_id_6) == blob_id) {

504

CS_SET_DISK_2(ptr.rp_ref->rr_type_2, MS_BLOB_FREE_REF);

505

if (free_ref)

506

free2_ref = free_ref;

507

free_ref = ptr.rp_blob_ref;

508

}

509

}

510

else if (tab_index == INVALID_INDEX) {

511

/* The is a reference from the temporary log only!! */

512

if (free_ref)

513

free2_ref = free_ref;

514

free_ref = ptr.rp_blob_ref;

515

}

516

break;

517

}

518

default: { // Must be a blob REF, check that the BLOB reference doesn't already exist.

519

u_int tab_index;

520

tab_index = CS_GET_DISK_2(ptr.rp_blob_ref->er_table_2);

521

522

if (tab_index && tab_index < ref_count) {

523

MSRepoTableRefPtr tr;

524

525

tab_index--;

526

tr = (MSRepoTableRefPtr) (otab->myOTBuffer + myRepo->getRepoBlobHeadSize() + tab_index * ref_size);

527

if (CS_GET_DISK_4(tr->tr_table_id_4) == tab_id &&

528

CS_GET_DISK_6(tr->tr_blob_id_6) == blob_id) {

529

if (COMMIT_MASK(CS_GET_DISK_8(ptr.rp_blob_ref->er_blob_ref_id_8)) == blob_ref_id) {

530

char message[100];

531

snprintf(message, 100, "Duplicate BLOB reference: db_id: %"PRIu32", tab_id:%"PRIu32", blob_ref_id: %"PRIu64"\n", myRepo->myRepoDatabase->myDatabaseID, tab_id, blob_ref_id);

532

/* The reference already exists so there is nothing to do... */

533

self->myException.log(self, message);

534

goto done;

535

}

536

}

537

}

538

break;

539

}

540

}

541

ptr.rp_chars += ref_size;

542

size -= ref_size;

543

}

544

545

// A BLOB reference needs to be added and if there is not

546

// already a table reference then a table reference must be added

547

// also.

548

if (!free_ref || (!tab_ref && !free2_ref)) {

549

size_t new_refs = (tab_ref)?1:2;

550

ptr.rp_chars = otab->myOTBuffer;

551

int sp = MS_VAR_SPACE(ptr.rp_head);

552

553

if (sp > (new_refs * CS_GET_DISK_1(ptr.rp_head->rb_ref_size_1))) {

554

sp = MS_MIN_BLOB_HEAD_SIZE;

555

}

556

557

if (MS_CAN_ADD_REFS(ptr.rp_head, new_refs)) {

558

new_head_size = head_size;

559

560

} else { // The header must be grown

561

size_t new_size, max_refs;

562

563

if (ref_count < 2)

564

max_refs = 4;

565

else if (ref_count > 32)

566

max_refs = ref_count + 32;

567

else

568

max_refs = 2 * ref_count;

569

570

if (max_refs > (MS_OT_BUFFER_SIZE/ref_size))

571

max_refs = (MS_OT_BUFFER_SIZE/ref_size);

572

573

if (max_refs < (ref_count + new_refs))

574

CSException::throwAssertion(CS_CONTEXT, "BLOB reference header overflow");

575

576

new_size = head_size + ref_size * max_refs;

577

578

//Shift the metadata in the header

579

if (CS_GET_DISK_2(ptr.rp_head->rb_mdata_size_2)) {

580

uint16_t mdata_size, mdata_offset, alias_offset, shift;

581

582

shift = new_size - head_size;

583

mdata_size = CS_GET_DISK_2(ptr.rp_head->rb_mdata_size_2);

584

mdata_offset = CS_GET_DISK_2(ptr.rp_head->rb_mdata_offset_2);

585

alias_offset = CS_GET_DISK_2(ptr.rp_head->rb_alias_offset_2);

586

587

memmove(ptr.rp_chars + mdata_offset + shift, ptr.rp_chars + mdata_offset, shift);

588

memset(ptr.rp_chars + mdata_offset, 0, shift);

589

mdata_offset += shift;

590

alias_offset += shift;

591

592

CS_SET_DISK_2(ptr.rp_head->rb_mdata_offset_2, mdata_offset);

593

CS_SET_DISK_2(ptr.rp_head->rb_alias_offset_2, alias_offset);

594

595

} else

596

memset(ptr.rp_chars + head_size, 0, new_size - head_size);

597

598

new_head_size = new_size;

599

}

600

CS_SET_DISK_2(ptr.rp_head->rb_head_size_2, new_head_size);

601

CS_SET_DISK_2(ptr.rp_head->rb_ref_count_2, ref_count + new_refs);

602

ptr.rp_chars += myRepo->myRepoBlobHeadSize + ref_count * ref_size;

603

604

if (!free_ref) {

605

free_ref = ptr.rp_blob_ref;

606

memset(free_ref, 0, ref_size);

607

ptr.rp_chars += ref_size;

608

}

609

610

if (!tab_ref) {

611

free2_ref = ptr.rp_blob_ref;

612

memset(free2_ref, 0, ref_size);

613

}

614

615

ref_count += new_refs;

616

}

617

else

618

new_head_size = head_size;

619

620

if (!tab_ref) {

621

tab_ref = (MSRepoTableRefPtr) free2_ref;

622

623

CS_SET_DISK_2(tab_ref->rr_type_2, MS_BLOB_TABLE_REF);

624

CS_SET_DISK_4(tab_ref->tr_table_id_4, tab_id);

625

CS_SET_DISK_6(tab_ref->tr_blob_id_6, blob_id);

626

}

627

628

size_t tab_idx;

629

630

tab_idx = (((char *) tab_ref - otab->myOTBuffer) - myRepo->myRepoBlobHeadSize) / ref_size;

631

632

CS_SET_DISK_2(free_ref->er_table_2, tab_idx+1);

633

CS_SET_DISK_2(free_ref->er_col_index_2, col_index);

634

CS_SET_DISK_8(free_ref->er_blob_ref_id_8, UNCOMMITTED(blob_ref_id));

635

636

update_blob_header(otab, offset, blob_size, head_size, new_head_size);

637

#ifdef HAVE_ALIAS_SUPPORT

638

if (reset_alias_index)

639

myRepo->myRepoDatabase->registerBlobAlias(myRepo->myRepoID, offset, blob_alias);

640

#endif

641

642

done:

643

644

unlock_(lock);

645

exit_();

646

}

647

648

void MSRepoFile::setBlobMetaData(MSOpenTable *otab, uint64_t offset, const char *meta_data, uint16_t meta_data_len, bool reset_alias, const char *alias)

649

{

650

CSMutex *lock;

651

MSRepoPointersRec ptr;

652

size_t read_size;

653

uint16_t new_head_size;

654

uint64_t blob_size;

655

uint16_t head_size, mdata_size, mdata_offset, alias_offset = 0;

656

MSBlobHeadRec blob;

657

658

enter_();

659

/* Lock the BLOB: */

660

lock = &myRepo->myRepoLock[offset % CS_REPO_REC_LOCK_COUNT];

661

lock_(lock);

662

663

/* Read the header: */

664

if (read(&blob, offset, sizeof(MSBlobHeadRec), 0) < sizeof(MSBlobHeadRec)) {

665

CSException::throwException(CS_CONTEXT, MS_ERR_NOT_FOUND, "BLOB header incomplete");

666

}

667

668

head_size = CS_GET_DISK_2(blob.rb_head_size_2);

669

670

if (head_size > MS_OT_BUFFER_SIZE) {

671

CSException::throwAssertion(CS_CONTEXT, "BLOB header overflow");

672

}

673

674

read_size = read(otab->myOTBuffer, offset, head_size, 0);

675

ptr.rp_chars = otab->myOTBuffer;

676

if (CS_GET_DISK_4(ptr.rp_head->rd_magic_4) != MS_BLOB_HEADER_MAGIC)

677

CSException::throwException(CS_CONTEXT, MS_ERR_NOT_FOUND, "Invalid BLOB identifier");

678

if (read_size < myRepo->myRepoBlobHeadSize)

679

CSException::throwException(CS_CONTEXT, MS_ERR_NOT_FOUND, "BLOB header incomplete");

680

if (! IN_USE_BLOB_STATUS(CS_GET_DISK_1(ptr.rp_head->rb_status_1)))

681

CSException::throwException(CS_CONTEXT, MS_ERR_NOT_FOUND, "BLOB has already been deleted");

682

683

684

blob_size = CS_GET_DISK_6(ptr.rp_head->rb_blob_repo_size_6);

685

if (read_size < head_size) {

686

/* This should not happen, because the file has been recovered,

687

* which should have already adjusted the head and blob

688

* size.

689

* If this happens then the file must have been truncated an the BLOB has been

690

* lost so we set the blob size to zero.

691

692

head_size = read_size;

693

blob_size = 0;

694

}

695

mdata_size = CS_GET_DISK_2(ptr.rp_head->rb_mdata_size_2);

696

697

if ((meta_data_len < mdata_size) || MS_CAN_ADD_MDATA(ptr.rp_head, meta_data_len - mdata_size))

698

new_head_size = head_size;

699

else { // The header must be grown

700

701

new_head_size = head_size + meta_data_len - mdata_size;

702

if (new_head_size > MS_OT_BUFFER_SIZE)

703

CSException::throwAssertion(CS_CONTEXT, "BLOB reference header overflow");

704

705

memset(ptr.rp_chars + head_size, 0, new_head_size - head_size);

706

}

707

708

// Meta data is placed at the end of the header.

709

if (meta_data_len)

710

mdata_offset = new_head_size - meta_data_len;

711

else

712

mdata_offset = 0;

713

mdata_size = meta_data_len;

714

715

uint32_t alias_hash = INVALID_ALIAS_HASH;

716

717

CS_SET_DISK_2(ptr.rp_head->rb_mdata_size_2, mdata_size);

718

CS_SET_DISK_2(ptr.rp_head->rb_mdata_offset_2, mdata_offset);

719

720

#ifdef HAVE_ALIAS_SUPPORT

721

if (alias) {

722

alias_hash = CS_GET_DISK_4(ptr.rp_head->rb_alias_hash_4);

723

alias_offset = CS_GET_DISK_2(ptr.rp_head->rb_alias_offset_2);

724

if (reset_alias) {

725

if (alias_offset)

726

alias_hash = myRepo->myRepoDatabase->updateBlobAlias(myRepo->myRepoID, offset, alias_hash, alias);

727

else {

728

alias_hash = myRepo->myRepoDatabase->registerBlobAlias(myRepo->myRepoID, offset, alias);

729

}

730

}

731

alias_offset = mdata_offset + (alias - meta_data);

732

733

} else if (reset_alias && CS_GET_DISK_2(ptr.rp_head->rb_alias_offset_2)) {

734

alias_offset = CS_GET_DISK_2(ptr.rp_head->rb_alias_offset_2);

735

myRepo->myRepoDatabase->deleteBlobAlias(myRepo->myRepoID, offset, CS_GET_DISK_4(ptr.rp_head->rb_alias_hash_4));

736

alias_offset = 0;

737

}

738

#else

739

if (alias || reset_alias) {

740

CSException::throwException(CS_CONTEXT, MS_ERR_NOT_IMPLEMENTED, "No BLOB alias support.");

741

}

742

#endif

743

744

CS_SET_DISK_2(ptr.rp_head->rb_alias_offset_2, alias_offset);

745

CS_SET_DISK_4(ptr.rp_head->rb_alias_hash_4, alias_hash);

746

747

memcpy(ptr.rp_chars + mdata_offset, meta_data, meta_data_len);

748

749

update_blob_header(otab, offset, blob_size, head_size, new_head_size);

750

751

unlock_(lock);

752

exit_();

753

754

}

755

756

757

void MSRepoFile::releaseBlob(MSOpenTable *otab, uint64_t offset, uint16_t head_size, uint32_t tab_id, uint64_t blob_id, uint64_t blob_ref_id, uint32_t auth_code)

758

{

759

CSMutex *lock;

760

MSRepoPointersRec ptr;

761

u_int table_ref_count = 0;

762

u_int size;

763

size_t ref_size, ref_count, read_size;

764

MSRepoTempRefPtr temp_ref = NULL;

765

uint16_t tab_index;

766

MSRepoTableRefPtr tab_ref;

767

uint16_t alias_offset;

768

uint32_t alias_hash;

769

770

enter_();

771

/* Lock the BLOB: */

772

lock = &myRepo->myRepoLock[offset % CS_REPO_REC_LOCK_COUNT];

773

lock_(lock);

774

/* Read the header: */

775

ASSERT(head_size <= MS_OT_BUFFER_SIZE);

776

read_size = read(otab->myOTBuffer, offset, head_size, 0);

777

ptr.rp_chars = otab->myOTBuffer;

778

if (CS_GET_DISK_4(ptr.rp_head->rd_magic_4) != MS_BLOB_HEADER_MAGIC)

779

CSException::throwException(CS_CONTEXT, MS_ERR_NOT_FOUND, "Invalid BLOB identifier");

780

if (read_size < myRepo->myRepoBlobHeadSize) {

781

removeBlob(otab, tab_id, blob_id, offset, auth_code);

782

goto exit;

783

}

784

if ((! IN_USE_BLOB_STATUS(CS_GET_DISK_1(ptr.rp_head->rb_status_1))) ||

785

CS_GET_DISK_4(ptr.rp_bytes + myRepo->myRepoBlobHeadSize - 4) != auth_code) {

786

removeBlob(otab, tab_id, blob_id, offset, auth_code);

787

goto exit;

788

}

789

790

/* Assume that what is here is correct: */

791

if (head_size != CS_GET_DISK_2(ptr.rp_head->rb_head_size_2)) {

792

head_size = CS_GET_DISK_2(ptr.rp_head->rb_head_size_2);

793

read_size = read(otab->myOTBuffer, offset, head_size, myRepo->myRepoBlobHeadSize);

794

}

795

head_size = CS_GET_DISK_2(ptr.rp_head->rb_head_size_2);

796

if (read_size < head_size) {

797

/* This should not happen, because the file has been recovered,

798

* which should have already adjusted the head and blob

799

* size.

800

* If this happens then the file must have been truncated an the BLOB has been

801

* lost so we set the blob size to zero.

802

803

head_size = read_size;

804

}

805

ref_size = CS_GET_DISK_1(ptr.rp_head->rb_ref_size_1);

806

ref_count = CS_GET_DISK_2(ptr.rp_head->rb_ref_count_2);

807

808

alias_offset = CS_GET_DISK_2(ptr.rp_head->rb_alias_offset_2);

809

alias_hash = CS_GET_DISK_4(ptr.rp_head->rb_alias_hash_4);

810

811

size = head_size - myRepo->myRepoBlobHeadSize;

812

if (size > ref_size * ref_count)

813

size = ref_size * ref_count;

814

ptr.rp_chars += myRepo->myRepoBlobHeadSize;

815

while (size >= ref_size) {

816

switch (CS_GET_DISK_2(ptr.rp_ref->rr_type_2)) {

817

case MS_BLOB_FREE_REF:

818

case MS_BLOB_TABLE_REF:

819

break;

820

case MS_BLOB_DELETE_REF: {

821

u_int tabi;

822

823

tabi = CS_GET_DISK_2(ptr.rp_temp_ref->tp_del_ref_2);

824

if (tabi && tabi < ref_count) {

825

tabi--;

826

tab_ref = (MSRepoTableRefPtr) (otab->myOTBuffer + myRepo->myRepoBlobHeadSize + tabi * ref_size);

827

if (CS_GET_DISK_4(tab_ref->tr_table_id_4) == tab_id &&

828

CS_GET_DISK_6(tab_ref->tr_blob_id_6) == blob_id) {

829

/* This is an old free, take it out. */

830

// Barry: What happens to the record in the temp log associated with this ref

831

// that is waiting to free the BLOB?

832

// Answer: It will find that there is MS_BLOB_DELETE_REF record with the BLOB

833

// or if there is one it will be for a different free in a different temp log

834

// or with a different temp log offset.

835

CS_SET_DISK_2(ptr.rp_ref->rr_type_2, MS_BLOB_FREE_REF);

836

}

837

}

838

break;

839

}

840

default: // Must be a blob REF

841

tab_ref = (MSRepoTableRefPtr) (otab->myOTBuffer + myRepo->myRepoBlobHeadSize + (CS_GET_DISK_2(ptr.rp_blob_ref->er_table_2)-1) * ref_size);

842

if (CS_GET_DISK_4(tab_ref->tr_table_id_4) == tab_id &&

843

CS_GET_DISK_6(tab_ref->tr_blob_id_6) == blob_id) {

844

if (COMMIT_MASK(CS_GET_DISK_8(ptr.rp_blob_ref->er_blob_ref_id_8)) == blob_ref_id) {

845

/* Found the reference, remove it... */

846

tab_index = CS_GET_DISK_2(ptr.rp_blob_ref->er_table_2)-1;

847

temp_ref = ptr.rp_temp_ref;

848

//temp_ref = (MSRepoTempRefPtr) tab_ref; // Set temp ref to the table ref so that it will be removed if there are no more references to it.

849

CS_SET_DISK_2(ptr.rp_ref->rr_type_2, MS_BLOB_FREE_REF);

850

}

851

else

852

table_ref_count++;

853

}

854

break;

855

}

856

ptr.rp_chars += ref_size;

857

size -= ref_size;

858

}

859

860

// If the refernce was found and there are no

861

// table references then the BLOB can be scheduled for deletion.

862

if ((!table_ref_count) && temp_ref) {

863

uint32_t log_id;

864

uint32_t log_offset;

865

uint32_t temp_time;

866

#ifdef HAVE_ALIAS_SUPPORT

867

MSDiskAliasRec aliasDiskRec;

868

MSDiskAliasPtr aliasDiskPtr = NULL;

869

870

if (alias_offset) {

871

CS_SET_DISK_4(aliasDiskRec.ar_repo_id_4, myRepo->myRepoID);

872

CS_SET_DISK_8(aliasDiskRec.ar_offset_8, offset);

873

CS_SET_DISK_4(aliasDiskRec.ar_hash_4, alias_hash);

874

aliasDiskPtr = &aliasDiskRec;

875

}

876

877

myRepo->myRepoDatabase->queueForDeletion(otab, MS_TL_BLOB_REF, tab_id, blob_id, auth_code, &log_id, &log_offset, &temp_time, aliasDiskPtr);

878

#else

879

myRepo->myRepoDatabase->queueForDeletion(otab, MS_TL_BLOB_REF, tab_id, blob_id, auth_code, &log_id, &log_offset, &temp_time);

880

#endif

881

myRepo->myLastTempTime = temp_time;

882

CS_SET_DISK_2(temp_ref->rr_type_2, MS_BLOB_DELETE_REF);

883

CS_SET_DISK_2(temp_ref->tp_del_ref_2, tab_index+1);

884

CS_SET_DISK_4(temp_ref->tp_log_id_4, log_id);

885

CS_SET_DISK_4(temp_ref->tp_offset_4, log_offset);

886

887

CS_SET_DISK_1(ptr.rp_head->rb_status_1, MS_BLOB_ALLOCATED); // The BLOB is allocated but no longer referenced

888

}

889

if (temp_ref) {

890

/* The reason I do not write the header of the header, is because

891

* I want to handle the rb_last_access_4 being set at the

892

* same time!

893

894

write(otab->myOTBuffer + MS_BLOB_STAT_OFFS, offset + MS_BLOB_STAT_OFFS, head_size - MS_BLOB_STAT_OFFS);

895

} else {

896

char message[100];

897

snprintf(message, 100, "BLOB reference not found: db_id: %"PRIu32", tab_id:%"PRIu32", blob_ref_id: %"PRIu64"\n", myRepo->myRepoDatabase->myDatabaseID, tab_id, blob_ref_id);

898

/* The reference already exists so there is nothing to do... */

899

self->myException.log(self, message);

900

}

901

902

exit:

903

unlock_(lock);

904

exit_();

905

}

906

907

void MSRepoFile::commitBlob(MSOpenTable *otab, uint64_t offset, uint16_t head_size, uint32_t tab_id, uint64_t blob_id, uint64_t blob_ref_id, uint32_t auth_code)

908

{

909

CSMutex *lock;

910

MSRepoPointersRec ptr;

911

u_int size;

912

size_t ref_size, ref_count, read_size;

913

MSRepoTableRefPtr tab_ref;

914

915

enter_();

916

/* Lock the BLOB: */

917

lock = &myRepo->myRepoLock[offset % CS_REPO_REC_LOCK_COUNT];

918

lock_(lock);

919

/* Read the header: */

920

ASSERT(head_size <= MS_OT_BUFFER_SIZE);

921

read_size = read(otab->myOTBuffer, offset, head_size, 0);

922

ptr.rp_chars = otab->myOTBuffer;

923

if (CS_GET_DISK_4(ptr.rp_head->rd_magic_4) != MS_BLOB_HEADER_MAGIC)

924

CSException::throwException(CS_CONTEXT, MS_ERR_NOT_FOUND, "Invalid BLOB identifier");

925

926

927

if (read_size < myRepo->myRepoBlobHeadSize)

928

CSException::throwException(CS_CONTEXT, MS_ERR_NOT_FOUND, "BLOB header incomplete");

929

if ( ! IN_USE_BLOB_STATUS(CS_GET_DISK_1(ptr.rp_head->rb_status_1)))

930

CSException::throwException(CS_CONTEXT, MS_ERR_NOT_FOUND, "BLOB has already been deleted");

931

if (auth_code && CS_GET_DISK_4(ptr.rp_bytes + myRepo->myRepoBlobHeadSize - 4) != auth_code)

932

CSException::throwException(CS_CONTEXT, MS_ERR_NOT_FOUND, "BLOB data does not match reference");

933

934

935

/* Assume that what is here is correct: */

936

if (head_size != CS_GET_DISK_2(ptr.rp_head->rb_head_size_2)) {

937

head_size = CS_GET_DISK_2(ptr.rp_head->rb_head_size_2);

938

read_size = read(otab->myOTBuffer, offset, head_size, myRepo->myRepoBlobHeadSize);

939

}

940

941

head_size = CS_GET_DISK_2(ptr.rp_head->rb_head_size_2);

942

if (read_size < head_size) {

943

/* This should not happen, because the file has been recovered,

944

* which should have already adjusted the head and blob

945

* size.

946

* If this happens then the file must have been truncated an the BLOB has been

947

* lost so we set the blob size to zero.

948

949

head_size = read_size;

950

}

951

ref_size = CS_GET_DISK_1(ptr.rp_head->rb_ref_size_1);

952

ref_count = CS_GET_DISK_2(ptr.rp_head->rb_ref_count_2);

953

954

size = head_size - myRepo->myRepoBlobHeadSize;

955

if (size > ref_size * ref_count)

956

size = ref_size * ref_count;

957

ptr.rp_chars += myRepo->myRepoBlobHeadSize;

958

while (size >= ref_size) {

959

switch (CS_GET_DISK_2(ptr.rp_ref->rr_type_2)) {

960

case MS_BLOB_FREE_REF:

961

case MS_BLOB_TABLE_REF:

962

break;

963

case MS_BLOB_DELETE_REF: {

964

break;

965

}

966

default: // Must be a blob REF

967

tab_ref = (MSRepoTableRefPtr) (otab->myOTBuffer + myRepo->myRepoBlobHeadSize + (CS_GET_DISK_2(ptr.rp_blob_ref->er_table_2)-1) * ref_size);

968

if (CS_GET_DISK_4(tab_ref->tr_table_id_4) == tab_id &&

969

CS_GET_DISK_6(tab_ref->tr_blob_id_6) == blob_id) {

970

uint64_t ref_id = CS_GET_DISK_8(ptr.rp_blob_ref->er_blob_ref_id_8);

971

if (COMMIT_MASK(ref_id) == blob_ref_id) {

972

/* Found the reference, mark it as committed... */

973

CS_SET_DISK_8(ptr.rp_blob_ref->er_blob_ref_id_8, blob_ref_id);

974

offset += (ptr.rp_chars - otab->myOTBuffer) + offsetof(MSRepoBlobRefRec, er_blob_ref_id_8);

975

write(&(ptr.rp_blob_ref->er_blob_ref_id_8), offset, 8);

976

goto exit;

977

}

978

}

979

break;

980

}

981

ptr.rp_chars += ref_size;

982

size -= ref_size;

983

}

984

985

char message[100];

986

snprintf(message, 100, "BLOB reference not found: db_id: %"PRIu32", tab_id:%"PRIu32", blob_ref_id: %"PRIu64"\n", myRepo->myRepoDatabase->myDatabaseID, tab_id, blob_ref_id);

987

self->myException.log(self, message);

988

989

exit:

990

unlock_(lock);

991

exit_();

992

}

993

994

void MSRepoFile::realFreeBlob(MSOpenTable *otab, char *buffer, uint32_t auth_code, uint64_t offset, uint16_t head_size, uint64_t blob_size, size_t ref_size)

995

{

996

uint32_t tab_id;

997

uint64_t blob_id;

998

size_t size;

999

MSRepoPointersRec ptr;

1000

enter_();

1001

1002

ptr.rp_chars = buffer;

1003

1004

if (BLOB_IN_CLOUD(CS_GET_DISK_1(ptr.rp_head->rb_storage_type_1))) {

1005

CloudKeyRec key;

1006

getBlobKey(ptr.rp_head, &key);

1007

if (!myRepo->myRepoDatabase->myBlobCloud)

1008

CSException::throwException(CS_CONTEXT, CS_ERR_GENERIC_ERROR, "Deleting cloud BLOB without cloud.");

1009

1010

myRepo->myRepoDatabase->myBlobCloud->cl_deleteData(&key);

1011

}

1012

1013

#ifdef HAVE_ALIAS_SUPPORT

1014

uint32_t alias_hash;

1015

alias_hash = CS_GET_DISK_4(ptr.rp_head->rb_alias_hash_4);

1016

if (alias_hash != INVALID_ALIAS_HASH)

1017

myRepo->myRepoDatabase->deleteBlobAlias(myRepo->myRepoID, offset, alias_hash);

1018

#endif

1019

1020

// Assuming the BLOB is still locked:

1021

CS_SET_DISK_1(ptr.rp_head->rb_status_1, MS_BLOB_DELETED);

1022

write(ptr.rp_chars + MS_BLOB_STAT_OFFS, offset + MS_BLOB_STAT_OFFS, head_size - MS_BLOB_STAT_OFFS);

1023

1024

/* Update garbage count: */

1025

updateGarbage(head_size + blob_size);

1026

1027

/* Remove all table references (should not be any)! */

1028

size = head_size - myRepo->myRepoBlobHeadSize;

1029

ptr.rp_chars += myRepo->myRepoBlobHeadSize;

1030

while (size >= ref_size) {

1031

if (CS_GET_DISK_2(ptr.rp_ref->rr_type_2) == MS_BLOB_TABLE_REF) {

1032

tab_id = CS_GET_DISK_4(ptr.rp_tab_ref->tr_table_id_4);

1033

blob_id = CS_GET_DISK_6(ptr.rp_tab_ref->tr_blob_id_6);

1034

removeBlob(otab, tab_id, blob_id, offset, auth_code);

1035

}

1036

ptr.rp_chars += ref_size;

1037

size -= ref_size;

1038

}

1039

exit_();

1040

}

1041

1042

/* This function will free the BLOB reference, if the record is invalid. */

1043

void MSRepoFile::freeTableReference(MSOpenTable *otab, uint64_t offset, uint16_t head_size, uint32_t tab_id, uint64_t blob_id, uint32_t auth_code)

1044

{

1045

CSMutex *lock;

1046

MSRepoPointersRec ptr;

1047

u_int blob_ref_count = 0;

1048

u_int table_ref_count = 0;

1049

bool modified = false;

1050

u_int size;

1051

size_t ref_size, ref_count, read_size;

1052

MSRepoTableRefPtr tab_ref = NULL;

1053

uint64_t blob_size;

1054

1055

enter_();

1056

/* Lock the BLOB: */

1057

lock = &myRepo->myRepoLock[offset % CS_REPO_REC_LOCK_COUNT];

1058

lock_(lock);

1059

/* Read the header: */

1060

ASSERT(head_size <= MS_OT_BUFFER_SIZE);

1061

read_size = read(otab->myOTBuffer, offset, head_size, 0);

1062

ptr.rp_chars = otab->myOTBuffer;

1063

if (CS_GET_DISK_4(ptr.rp_head->rd_magic_4) != MS_BLOB_HEADER_MAGIC)

1064

CSException::throwException(CS_CONTEXT, MS_ERR_NOT_FOUND, "Invalid BLOB identifier");

1065

if (read_size < myRepo->myRepoBlobHeadSize) {

1066

removeBlob(otab, tab_id, blob_id, offset, auth_code);

1067

goto exit;

1068

}

1069

if ((! IN_USE_BLOB_STATUS(CS_GET_DISK_1(ptr.rp_head->rb_status_1))) ||

1070

CS_GET_DISK_4(ptr.rp_bytes + myRepo->myRepoBlobHeadSize - 4) != auth_code) {

1071

removeBlob(otab, tab_id, blob_id, offset, auth_code);

1072

goto exit;

1073

}

1074

1075

/* Assume that what is here is correct: */

1076

if (head_size != CS_GET_DISK_2(ptr.rp_head->rb_head_size_2)) {

1077

head_size = CS_GET_DISK_2(ptr.rp_head->rb_head_size_2);

1078

read_size = read(otab->myOTBuffer, offset, head_size, myRepo->myRepoBlobHeadSize);

1079

}

1080

head_size = CS_GET_DISK_2(ptr.rp_head->rb_head_size_2);

1081

blob_size = CS_GET_DISK_6(ptr.rp_head->rb_blob_repo_size_6);

1082

if (read_size < head_size) {

1083

/* This should not happen, because the file has been recovered,

1084

* which should have already adjusted the head and blob

1085

* size.

1086

* If this happens then the file must have been truncated an the BLOB has been

1087

* lost so we set the blob size to zero.

1088

1089

head_size = read_size;

1090

blob_size = 0;

1091

1092

}

1093

ref_size = CS_GET_DISK_1(ptr.rp_head->rb_ref_size_1);

1094

ref_count = CS_GET_DISK_2(ptr.rp_head->rb_ref_count_2);

1095

size = head_size - myRepo->myRepoBlobHeadSize;

1096

if (size > ref_size * ref_count)

1097

size = ref_size * ref_count;

1098

ptr.rp_chars += myRepo->myRepoBlobHeadSize;

1099

while (size >= ref_size) {

1100

switch (CS_GET_DISK_2(ptr.rp_ref->rr_type_2)) {

1101

case MS_BLOB_FREE_REF:

1102

break;

1103

case MS_BLOB_TABLE_REF:

1104

if (CS_GET_DISK_4(ptr.rp_tab_ref->tr_table_id_4) == tab_id &&

1105

CS_GET_DISK_6(ptr.rp_tab_ref->tr_blob_id_6) == blob_id)

1106

tab_ref = ptr.rp_tab_ref;

1107

break;

1108

case MS_BLOB_DELETE_REF:

1109

break;

1110

default:

1111

MSRepoTableRefPtr tr;

1112

1113

tr = (MSRepoTableRefPtr) (otab->myOTBuffer + myRepo->myRepoBlobHeadSize + (CS_GET_DISK_2(ptr.rp_blob_ref->er_table_2)-1) * ref_size);

1114

if (CS_GET_DISK_2(tr->rr_type_2) == MS_BLOB_TABLE_REF) {

1115

/* I am deleting all references of a table. So I am here to

1116

* also delete the blob references that refer to the

1117

* table reference!!!

1118

1119

if (CS_GET_DISK_4(tr->tr_table_id_4) == tab_id && CS_GET_DISK_6(tr->tr_blob_id_6) == blob_id) {

1120

/* Free the blob reference: */

1121

CS_SET_DISK_2(ptr.rp_ref->rr_type_2, MS_BLOB_FREE_REF);

1122

modified = true;

1123

}

1124

else

1125

blob_ref_count++;

1126

1127

}

1128

break;

1129

}

1130

ptr.rp_chars += ref_size;

1131

size -= ref_size;

1132

}

1133

1134

if (!table_ref_count && tab_ref) {

1135

CS_SET_DISK_2(tab_ref->rr_type_2, MS_BLOB_FREE_REF);

1136

modified = true;

1137

}

1138

1139

1140

if (!blob_ref_count) {

1141

realFreeBlob(otab, otab->myOTBuffer, auth_code, offset, head_size, blob_size, ref_size);

1142

} else if (modified)

1143

/* The reason I do not write the header of the header, is because

1144

* I want to handle the rb_last_access_4 being set at the

1145

* same time!

1146

1147

write(otab->myOTBuffer + MS_BLOB_STAT_OFFS, offset + MS_BLOB_STAT_OFFS, head_size - MS_BLOB_STAT_OFFS);

1148

1149

unlock_(lock);

1150

1151

if (!table_ref_count || !tab_ref)

1152

/* Free the table reference, if there are no more

1153

* blob references, reference the table reference,

1154

* or if the table reference was not found in the

1155

* BLOB at all!

1156

1157

removeBlob(otab, tab_id, blob_id, offset, auth_code);

1158

1159

exit_();

1160

1161

exit:

1162

unlock_(lock);

1163

exit_();

1164

}

1165

1166

void MSRepoFile::checkBlob(MSOpenTable *otab, CSStringBuffer *buffer, uint64_t offset, uint32_t auth_code, uint32_t temp_log_id, uint32_t temp_log_offset)

1167

{

1168

CSMutex *lock;

1169

MSBlobHeadRec blob;

1170

MSRepoPointersRec ptr;

1171

u_int blob_ref_count = 0;

1172

bool modified = false;

1173

u_int size;

1174

size_t ref_size, ref_count, read_size;

1175

uint8_t status;

1176

uint16_t head_size;

1177

uint64_t blob_size;

1178

MSRepoTempRefPtr my_ref = NULL;

1179

uint16_t ref_type = MS_BLOB_FREE_REF;

1180

enter_();

1181

1182

/* Lock the BLOB: */

1183

lock = &myRepo->myRepoLock[offset % CS_REPO_REC_LOCK_COUNT];

1184

lock_(lock);

1185

1186

/* Read the head of the header: */

1187

if (read(&blob, offset, sizeof(MSBlobHeadRec), 0) < sizeof(MSBlobHeadRec))

1188

goto exit;

1189

1190

// Because the temp log will be replayed from the start when the server

1191

// is restarted it is likely that it will have references to BLOBs that

1192

// no longer exist. So it is not an error if the BLOB ref doesn't point to

1193

// a valid BLOB.

1194

1195

// At some point this should probably be rethought because you cannot

1196

// tell the diference between a bad ref because of old data and a bad

1197

// ref because of a BUG.

1198

if (CS_GET_DISK_4(blob.rd_magic_4) != MS_BLOB_HEADER_MAGIC)

1199

goto exit;

1200

1201

head_size = CS_GET_DISK_2(blob.rb_head_size_2);

1202

blob_size = CS_GET_DISK_6(blob.rb_blob_repo_size_6);

1203

ref_size = CS_GET_DISK_1(blob.rb_ref_size_1);

1204

ref_count = CS_GET_DISK_2(blob.rb_ref_count_2);

1205

status = CS_GET_DISK_1(blob.rb_status_1);

1206

if (! IN_USE_BLOB_STATUS(status))

1207

goto exit;

1208

1209

/* Read the entire header: */

1210

buffer->setLength(head_size);

1211

ptr.rp_chars = buffer->getBuffer(0);

1212

read_size = read(ptr.rp_chars, offset, head_size, 0);

1213

if (read_size < myRepo->myRepoBlobHeadSize)

1214

goto exit;

1215

if (CS_GET_DISK_4(ptr.rp_bytes + myRepo->myRepoBlobHeadSize - 4) != auth_code)

1216

goto exit;

1217

if (read_size < head_size) {

1218

/* This should not happen, because the file has been recovered,

1219

* which should have already adjusted the head and blob

1220

* size.

1221

* If this happens then the file must have been truncated an the BLOB has been

1222

* lost so we set the blob size to zero.

1223

1224

head_size = read_size;

1225

blob_size = 0;

1226

}

1227

size = head_size - myRepo->myRepoBlobHeadSize;

1228

if (size > ref_size * ref_count)

1229

size = ref_size * ref_count;

1230

1231

1232

/* Search through all references: */

1233

ptr.rp_chars += myRepo->myRepoBlobHeadSize;

1234

while (size >= ref_size) {

1235

switch (CS_GET_DISK_2(ptr.rp_ref->rr_type_2)) {

1236

case MS_BLOB_FREE_REF:

1237

break;

1238

case MS_BLOB_TABLE_REF:

1239

break;

1240

case MS_BLOB_DELETE_REF:

1241

if (CS_GET_DISK_4(ptr.rp_temp_ref->tp_log_id_4) == temp_log_id &&

1242

CS_GET_DISK_4(ptr.rp_temp_ref->tp_offset_4) == temp_log_offset) {

1243

ref_type = CS_GET_DISK_2(ptr.rp_ref->rr_type_2);

1244

my_ref = ptr.rp_temp_ref;

1245

CS_SET_DISK_2(ptr.rp_ref->rr_type_2, MS_BLOB_FREE_REF);

1246

modified = true;

1247

}

1248

break;

1249

default:

1250

MSRepoTableRefPtr tr;

1251

u_int tabi;

1252

1253

tabi = CS_GET_DISK_2(ptr.rp_blob_ref->er_table_2);

1254

if (tabi < ref_count) {

1255

tr = (MSRepoTableRefPtr) (buffer->getBuffer(0) + myRepo->myRepoBlobHeadSize + (tabi-1) * ref_size);

1256

if (CS_GET_DISK_2(tr->rr_type_2) == MS_BLOB_TABLE_REF)

1257

blob_ref_count++;

1258

}

1259

break;

1260

}

1261

ptr.rp_chars += ref_size;

1262

size -= ref_size;

1263

}

1264

1265

if ((ref_type == (uint16_t)MS_BLOB_DELETE_REF) && !blob_ref_count) {

1266

realFreeBlob(NULL, buffer->getBuffer(0), auth_code, offset, head_size, blob_size, ref_size);

1267

}

1268

1269

exit:

1270

unlock_(lock);

1271

exit_();

1272

}

1273

1274

void MSRepoFile::returnToPool()

1275

{

1276

myRepo->myRepoDatabase->returnRepoFileToPool(this);

1277

}

1278

1279

void MSRepoFile::removeBlob(MSOpenTable *otab, uint32_t tab_id, uint64_t blob_id, uint64_t offset, uint32_t auth_code)

1280

{

1281

enter_();

1282

if (otab && otab->getDBTable()->myTableID == tab_id)

1283

otab->getDBTable()->freeBlobHandle(otab, blob_id, myRepo->myRepoID, offset, auth_code);

1284

else {

1285

MSOpenTable *tmp_otab;

1286

1287

if ((tmp_otab = MSTableList::getOpenTableByID(myRepo->myRepoDatabase->myDatabaseID, tab_id))) {

1288

frompool_(tmp_otab);

1289

tmp_otab->getDBTable()->freeBlobHandle(tmp_otab, blob_id, myRepo->myRepoID, offset, auth_code);

1290

backtopool_(tmp_otab);

1291

}

1292

}

1293

exit_();

1294

}

1295

1296

MSRepoFile *MSRepoFile::newRepoFile(MSRepository *repo, CSPath *path)

1297

{

1298

MSRepoFile *f;

1299

1300

if (!(f = new MSRepoFile())) {

1301

path->release();

1302

CSException::throwOSError(CS_CONTEXT, ENOMEM);

1303

}

1304

f->myRepo = repo;

1305

f->myFilePath = path;

1306

return f;

1307

}

1308

1309

1310

* ---------------------------------------------------------------

1311

* REPOSITORY

1312

1313

1314

MSRepository::MSRepository(u_int id, MSDatabase *db, off_t file_size):

1315

CSSharedRefObject(),

1316

myRepoID(id),

1317

myRepoFileSize(file_size),

1318

myRepoXLock(false),

1319

myRepoLockState(REPO_UNLOCKED),

1320

isRemovingFP(false),

1321

myRepoDatabase(db),

1322

myGarbageCount(0),

1323

myRepoHeadSize(0),

1324

myRepoDefRefSize(0),

1325

myRepoBlobHeadSize(0),

1326

myRecoveryOffset(0),

1327

myLastTempTime(0),

1328

myLastAccessTime(0),

1329

myLastCreateTime(0),

1330

myLastRefTime(0),

1331

mustBeDeleted(false),

1332

iFilePool(NULL)

1333

{

1334

}

1335

1336

MSRepository::~MSRepository()

1337

{

1338

CSPath *path = NULL;

1339

1340

enter_();

1341

if (mustBeDeleted) {

1342

path = getRepoFilePath();

1343

push_(path);

1344

}

1345

1346

isRemovingFP = true;

1347

removeRepoFilesNotInUse();

1348

/* With this, I also delete those that are in use!: */

1349

iPoolFiles.clear();

1350

1351

if (path) {

1352

path->removeFile();

1353

release_(path);

1354

}

1355

exit_();

1356

}

1357

1358

void MSRepository::openRepoFileForWriting(MSOpenTable *otab)

1359

{

1360

if (!otab->myWriteRepoFile)

1361

otab->myWriteRepoFile = openRepoFile();

1362

}

1363

1364

uint64_t MSRepository::receiveBlob(MSOpenTable *otab, uint16_t head_size, uint64_t blob_size, Md5Digest *checksum, CSInputStream *stream)

1365

{

1366

off_t offset;

1367

size_t tfer;

1368

1369

enter_();

1370

1371

offset = myRepoFileSize;

1372

1373

offset += head_size;

1374

1375

ASSERT(myRepoDatabase->myBlobType == MS_STANDARD_STORAGE);

1376

if (stream) {

1377

CSMd5 md5;

1378

push_(stream);

1379

md5.md5_init();

1380

while (blob_size > 0) {

1381

if (blob_size <= MS_OT_BUFFER_SIZE)

1382

tfer = (size_t) blob_size;

1383

else

1384

tfer = MS_OT_BUFFER_SIZE;

1385

tfer = stream->read(otab->myOTBuffer, tfer);

1386

if (!tfer)

1387

CSException::throwOSError(CS_CONTEXT, EPIPE);

1388

if (checksum) md5.md5_append((const u_char *)(otab->myOTBuffer), tfer);

1389

otab->myWriteRepoFile->write(otab->myOTBuffer, offset, tfer);

1390

offset += (uint64_t) tfer;

1391

blob_size -= (uint64_t) tfer;

1392

}

1393

if (checksum) md5.md5_digest(checksum);

1394

release_(stream);

1395

} else {

1396

// Write 1 byte to the end to reserver the space.

1397

otab->myWriteRepoFile->write("x" , offset + blob_size -1, 1);

1398

}

1399

1400

return_( myRepoFileSize);

1401

}

1402

1403

// copyBlob() copies the BLOB and its header.

1404

uint64_t MSRepository::copyBlob(MSOpenTable *otab, uint64_t size, CSInputStream *stream)

1405

{

1406

off_t offset = myRepoFileSize;

1407

size_t tfer;

1408

1409

while (size > 0) {

1410

if (size <= MS_OT_BUFFER_SIZE)

1411

tfer = (size_t) size;

1412

else

1413

tfer = MS_OT_BUFFER_SIZE;

1414

tfer = stream->read(otab->myOTBuffer, tfer);

1415

if (!tfer)

1416

CSException::throwOSError(CS_CONTEXT, EPIPE);

1417

otab->myWriteRepoFile->write(otab->myOTBuffer, offset, tfer);

1418

offset += (uint64_t) tfer;

1419

size -= (uint64_t) tfer;

1420

}

1421

1422

return myRepoFileSize;

1423

}

1424

1425

void MSRepository::writeBlobHead(MSOpenTable *otab, uint64_t offset, uint8_t ref_size, uint16_t head_size, uint64_t blob_size, Md5Digest *checksum, char *metadata, uint16_t metadata_size, uint64_t blob_id, uint32_t auth_code, uint32_t log_id, uint32_t log_offset, uint8_t blob_type, CloudKeyPtr cloud_key)

1426

{

1427

MSBlobHeadPtr blob ;

1428

MSRepoTableRefPtr tab_ref;

1429

MSRepoTempRefPtr temp_ref;

1430

time_t now;

1431

uint16_t tab_idx, max_ref_count = (head_size - myRepoBlobHeadSize - metadata_size) / ref_size;

1432

size_t size;

1433

1434

if (max_ref_count > MS_REPO_MIN_REF_COUNT)

1435

max_ref_count = MS_REPO_MIN_REF_COUNT;

1436

1437

ASSERT(max_ref_count > 1);

1438

1439

if (blob_type == MS_CLOUD_STORAGE)

1440

now = cloud_key->creation_time;

1441

else

1442

now = time(NULL);

1443

1444

blob = (MSBlobHeadPtr) otab->myOTBuffer;

1445

CS_SET_DISK_4(blob->rb_last_access_4, now);

1446

CS_SET_DISK_4(blob->rb_mod_time_4, now);

1447

CS_SET_DISK_4(blob->rb_access_count_4, 0);

1448

CS_SET_DISK_4(blob->rb_backup_id_4, 0);

1449

CS_SET_DISK_4(blob->rb_create_time_4, now);

1450

CS_SET_DISK_4(blob->rd_magic_4, MS_BLOB_HEADER_MAGIC);

1451

CS_SET_DISK_2(blob->rb_head_size_2, head_size);

1452

CS_SET_DISK_6(blob->rb_blob_data_size_6, blob_size);

1453

CS_SET_DISK_1(blob->rb_status_1, MS_BLOB_ALLOCATED);

1454

CS_SET_DISK_1(blob->rb_ref_size_1, ref_size);

1455

CS_SET_DISK_2(blob->rb_ref_count_2, max_ref_count);

1456

CS_SET_DISK_4(blob->rb_last_ref_4, 0);

1457

CS_SET_DISK_4(otab->myOTBuffer + myRepoBlobHeadSize - 4, auth_code);

1458

if (checksum)

1459

memcpy(&(blob->rb_blob_checksum_md5d), checksum, sizeof(Md5Digest));

1460

1461

CS_SET_DISK_2(blob->rb_mdata_size_2, metadata_size);

1462

if (metadata_size) {

1463

uint16_t metadata_offset = head_size - metadata_size;

1464

1465

CS_SET_DISK_2(blob->rb_mdata_offset_2, metadata_offset);

1466

memcpy(otab->myOTBuffer + metadata_offset, metadata, metadata_size);

1467

1468

#ifdef HAVE_ALIAS_SUPPORT

1469

MetaData md;

1470

md.use_data(metadata, metadata_size);

1471

const char *alias;

1472

alias = md.findAlias();

1473

if (alias) {

1474

uint32_t alias_hash;

1475

uint16_t alias_offset = metadata_offset + (uint16_t) (alias - metadata);

1476

CS_SET_DISK_2(blob->rb_alias_offset_2, alias_offset);

1477

alias_hash = myRepoDatabase->registerBlobAlias(myRepoID, offset, alias);

1478

CS_SET_DISK_4(blob->rb_alias_hash_4, alias_hash);

1479

} else {

1480

CS_SET_DISK_2(blob->rb_alias_offset_2, 0);

1481

}

1482

#else

1483

CS_SET_DISK_2(blob->rb_alias_offset_2, 0);

1484

#endif

1485

1486

} else {

1487

CS_SET_DISK_2(blob->rb_mdata_offset_2, 0);

1488

CS_SET_DISK_2(blob->rb_alias_offset_2, 0);

1489

}

1490

1491

1492

if (blob_id) {

1493

tab_ref = (MSRepoTableRefPtr) (otab->myOTBuffer + myRepoBlobHeadSize);

1494

CS_SET_DISK_2(tab_ref->rr_type_2, MS_BLOB_TABLE_REF);

1495

CS_SET_DISK_4(tab_ref->tr_table_id_4, otab->getDBTable()->myTableID);

1496

CS_SET_DISK_6(tab_ref->tr_blob_id_6, blob_id);

1497

temp_ref = (MSRepoTempRefPtr) (otab->myOTBuffer + myRepoBlobHeadSize + ref_size);

1498

tab_idx = 1; // This is the index of the blob table ref in the repository record.

1499

size = myRepoBlobHeadSize + ref_size + ref_size;

1500

}

1501

else {

1502

temp_ref = (MSRepoTempRefPtr) (otab->myOTBuffer + myRepoBlobHeadSize);

1503

tab_idx = INVALID_INDEX; // Means not used

1504

size = myRepoBlobHeadSize + ref_size;

1505

}

1506

1507

CS_SET_DISK_2(temp_ref->rr_type_2, MS_BLOB_DELETE_REF);

1508

CS_SET_DISK_2(temp_ref->tp_del_ref_2, tab_idx);

1509

CS_SET_DISK_4(temp_ref->tp_log_id_4, log_id);

1510

CS_SET_DISK_4(temp_ref->tp_offset_4, log_offset);

1511

1512

if (blob_type == MS_CLOUD_STORAGE) { // The data is stored in the cloud and not in the repository.

1513

CS_SET_DISK_4(blob->rb_s3_key_id_4, cloud_key->ref_index);

1514

CS_SET_DISK_4(blob->rb_s3_cloud_ref_4, cloud_key->cloud_ref);

1515

blob_size = 0; // The blob is not stored in the repository so the blob storage size in the repository is zero

1516

}

1517

1518

memset(otab->myOTBuffer + size, 0, head_size - size - metadata_size);

1519

1520

CS_SET_DISK_1(blob->rb_storage_type_1, blob_type);

1521

CS_SET_DISK_6(blob->rb_blob_repo_size_6, blob_size);

1522

otab->myWriteRepoFile->write(blob, offset, head_size);

1523

1524

setRepoFileSize(otab, offset + head_size + blob_size);

1525

}

1526

1527

// This is called to update the header info for a new unreferenced blob that was

1528

// created as the result of the BLOB being copied from one database to another.

1529

// When this happens the BLOB and meta data is copied but the header is not set.

1530

void MSRepository::resetBlobHead(MSOpenTable *otab, uint64_t offset, uint16_t head_size, uint64_t blob_id, uint64_t blob_ref_id, uint32_t auth_code, uint16_t col_index, uint8_t blob_type)

1531

{

1532

MSBlobHeadPtr blob ;

1533

MSRepoTableRefPtr tab_ref;

1534

MSRepoBlobRefPtr blob_ref;

1535

uint16_t metadata_size;

1536

uint8_t ref_size;

1537

size_t size;

1538

1539

otab->myWriteRepoFile->read(otab->myOTBuffer, offset, head_size, head_size);

1540

1541

blob = (MSBlobHeadPtr) otab->myOTBuffer;

1542

1543

if (CS_GET_DISK_4(blob->rd_magic_4) != MS_BLOB_HEADER_MAGIC)

1544

CSException::throwException(CS_CONTEXT, MS_ERR_NOT_FOUND, "Invalid BLOB identifier");

1545

metadata_size = CS_GET_DISK_2(blob->rb_mdata_size_2);

1546

ref_size = CS_GET_DISK_1(blob->rb_ref_size_1);

1547

1548

CS_SET_DISK_4(blob->rb_last_ref_4, (uint32_t) time(NULL)); // Set the reference time

1549

CS_SET_DISK_1(blob->rb_status_1, MS_BLOB_REFERENCED);

1550

CS_SET_DISK_4(otab->myOTBuffer + myRepoBlobHeadSize - 4, auth_code);

1551

1552

// Add the table reference:

1553

tab_ref = (MSRepoTableRefPtr) (otab->myOTBuffer + myRepoBlobHeadSize);

1554

CS_SET_DISK_2(tab_ref->rr_type_2, MS_BLOB_TABLE_REF);

1555

CS_SET_DISK_4(tab_ref->tr_table_id_4, otab->getDBTable()->myTableID);

1556

CS_SET_DISK_6(tab_ref->tr_blob_id_6, blob_id);

1557

1558

// Add the blob reference:

1559

blob_ref = (MSRepoBlobRefPtr) (otab->myOTBuffer + myRepoBlobHeadSize + ref_size);

1560

CS_SET_DISK_2(blob_ref->er_table_2, 1);

1561

CS_SET_DISK_2(blob_ref->er_col_index_2, col_index);

1562

CS_SET_DISK_8(blob_ref->er_blob_ref_id_8, blob_ref_id);

1563

1564

size = myRepoBlobHeadSize + ref_size + ref_size;

1565

1566

// Any downloading of data from the cloud or copying

1567

// data from one cloud to another is assumed to have been

1568

// done before this.

1569

1570

memset(otab->myOTBuffer + size, 0, head_size - size - metadata_size); // Clear the unused reference slots.

1571

otab->myWriteRepoFile->write(blob, offset, head_size);

1572

}

1573

1574

void MSRepository::setRepoFileSize(MSOpenTable *otab, off_t offset)

1575

{

1576

myRepoFileSize = offset;

1577

if (myRepoFileSize >= MSDatabase::gRepoThreshold

1578

/**/ || getGarbageLevel() >= MSRepository::gGarbageThreshold)

1579

otab->closeForWriting();

1580

}

1581

1582

void MSRepository::syncHead(MSRepoFile *fh)

1583

{

1584

MSRepoHeadRec head;

1585

1586

fh->sync();

1587

myRecoveryOffset = myRepoFileSize;

1588

CS_SET_DISK_8(head.rh_recovery_offset_8, myRecoveryOffset);

1589

CS_SET_DISK_4(head.rh_last_temp_time_4, myLastTempTime);

1590

CS_SET_DISK_4(head.rh_last_access_4, myLastAccessTime);

1591

CS_SET_DISK_4(head.rh_create_time_4, myLastCreateTime);

1592

CS_SET_DISK_4(head.rh_last_ref_4, myLastRefTime);

1593

1594

fh->write(&head.rh_recovery_offset_8, offsetof(MSRepoHeadRec, rh_recovery_offset_8), 24);

1595

fh->sync();

1596

}

1597

1598

MSRepoFile *MSRepository::openRepoFile()

1599

{

1600

MSRepoFile *fh;

1601

1602

enter_();

1603

fh = MSRepoFile::newRepoFile(this, getRepoFilePath());

1604

push_(fh);

1605

if (myRepoFileSize)

1606

fh->open(CSFile::DEFAULT);

1607

else

1608

fh->open(CSFile::CREATE);

1609

if (!myRepoHeadSize) {

1610

MSRepoHeadRec head;

1611

MSBlobHeadRec blob;

1612

size_t size;

1613

int status;

1614

int ref_size;

1615

uint16_t head_size;

1616

uint64_t blob_size;

1617

1618

lock_(this);

1619

/* Check again after locking: */

1620

if (!myRepoHeadSize) {

1621

if (fh->read(&head, 0, offsetof(MSRepoHeadRec, rh_reserved_4), 0) < offsetof(MSRepoHeadRec, rh_reserved_4)) {

1622

CS_SET_DISK_4(head.rh_magic_4, MS_REPO_FILE_MAGIC);

1623

CS_SET_DISK_2(head.rh_version_2, MS_REPO_FILE_VERSION);

1624

CS_SET_DISK_2(head.rh_repo_head_size_2, MS_REPO_FILE_HEAD_SIZE);

1625

CS_SET_DISK_2(head.rh_blob_head_size_2, sizeof(MSBlobHeadRec));

1626

CS_SET_DISK_2(head.rh_def_ref_size_2, sizeof(MSRepoGenericRefRec));

1627

CS_SET_DISK_8(head.rh_recovery_offset_8, MS_REPO_FILE_HEAD_SIZE);

1628

CS_SET_DISK_8(head.rh_garbage_count_8, 0);

1629

CS_SET_DISK_4(head.rh_last_temp_time_4, 0);

1630

CS_SET_DISK_4(head.rh_last_access_4, 0);

1631

CS_SET_DISK_4(head.rh_create_time_4, 0);

1632

CS_SET_DISK_4(head.rh_last_ref_4, 0);

1633

CS_SET_DISK_4(head.rh_reserved_4, 0);

1634

fh->write(&head, 0, sizeof(MSRepoHeadRec));

1635

}

1636

1637

/* Check the file header: */

1638

if (CS_GET_DISK_4(head.rh_magic_4) != MS_REPO_FILE_MAGIC)

1639

CSException::throwFileError(CS_CONTEXT, fh->getPathString(), CS_ERR_BAD_HEADER_MAGIC);

1640

if (CS_GET_DISK_2(head.rh_version_2) > MS_REPO_FILE_VERSION)

1641

CSException::throwFileError(CS_CONTEXT, fh->getPathString(), CS_ERR_VERSION_TOO_NEW);

1642

1643

/* Load the header details: */

1644

myRepoHeadSize = CS_GET_DISK_2(head.rh_repo_head_size_2);

1645

myRepoDefRefSize = CS_GET_DISK_2(head.rh_def_ref_size_2);

1646

myRepoBlobHeadSize = CS_GET_DISK_2(head.rh_blob_head_size_2);

1647

myRecoveryOffset = CS_GET_DISK_8(head.rh_recovery_offset_8);

1648

myGarbageCount = CS_GET_DISK_8(head.rh_garbage_count_8);

1649

myLastTempTime = CS_GET_DISK_4(head.rh_last_temp_time_4);

1650

myLastAccessTime = CS_GET_DISK_4(head.rh_last_access_4);

1651

myLastCreateTime = CS_GET_DISK_4(head.rh_create_time_4);

1652

myLastRefTime = CS_GET_DISK_4(head.rh_last_ref_4);

1653

1654

/* File size, cannot be less than header size: */

1655

if (myRepoFileSize < myRepoHeadSize)

1656

myRepoFileSize = myRepoHeadSize;

1657

1658

ASSERT(myGarbageCount <= myRepoFileSize);

1659

1660

/* Recover the file: */

1661

while (myRecoveryOffset < myRepoFileSize) {

1662

if ((size = fh->read(&blob, myRecoveryOffset, MS_MIN_BLOB_HEAD_SIZE, 0)) < MS_MIN_BLOB_HEAD_SIZE) {

1663

if (size != 0) {

1664

myRepoFileSize = myRecoveryOffset;

1665

fh->setEOF(myRepoFileSize);

1666

}

1667

break;

1668

}

1669

uint16_t ref_count, mdata_size, mdata_offset;

1670

1671

status = CS_GET_DISK_1(blob.rb_status_1);

1672

ref_size = CS_GET_DISK_1(blob.rb_ref_size_1);

1673

ref_count = CS_GET_DISK_2(blob.rb_ref_count_2);

1674

mdata_size = CS_GET_DISK_2(blob.rb_mdata_size_2);

1675

mdata_size = CS_GET_DISK_2(blob.rb_mdata_size_2);

1676

mdata_offset = CS_GET_DISK_2(blob.rb_mdata_offset_2);

1677

blob_size = CS_GET_DISK_6(blob.rb_blob_repo_size_6);

1678

if ((CS_GET_DISK_4(blob.rd_magic_4) != MS_BLOB_HEADER_MAGIC) ||

1679

(! IN_USE_BLOB_STATUS(status)) ||

1680

head_size < (myRepoBlobHeadSize + ref_size * MS_REPO_MIN_REF_COUNT) ||

1681

head_size < (mdata_offset + mdata_size) ||

1682

((blob_size == 0) && (BLOB_IN_REPOSITORY(CS_GET_DISK_1(blob.rb_storage_type_1)))) ||

1683

myRecoveryOffset + head_size + blob_size > myRepoFileSize) {

1684

myRepoFileSize = myRecoveryOffset;

1685

fh->setEOF(myRepoFileSize);

1686

break;

1687

}

1688

myRecoveryOffset += head_size + blob_size;

1689

}

1690

1691

fh->sync();

1692

myRecoveryOffset = myRepoFileSize;

1693

CS_SET_DISK_8(head.rh_recovery_offset_8, myRecoveryOffset);

1694

fh->write(&head, offsetof(MSRepoHeadRec, rh_recovery_offset_8), 8);

1695

fh->sync();

1696

}

1697

unlock_(this);

1698

}

1699

pop_(fh);

1700

return_(fh);

1701

}

1702

1703

void MSRepository::lockRepo(RepoLockState state)

1704

{

1705

CSMutex *lock;

1706

enter_();

1707

1708

lock = &myRepoLock[0];

1709

lock_(lock);

1710

1711

ASSERT(!myRepoXLock);

1712

1713

myRepoLockState = state;

1714

myRepoXLock = true;

1715

1716

unlock_(lock);

1717

exit_();

1718

}

1719

1720

void MSRepository::signalCompactor()

1721

{

1722

#ifndef MS_COMPACTOR_POLLS

1723

if (!mustBeDeleted) {

1724

if (getGarbageLevel() >= MSRepository::gGarbageThreshold) {

1725

if (myRepoDatabase->myCompactorThread)

1726

myRepoDatabase->myCompactorThread->wakeup();

1727

}

1728

}

1729

#endif

1730

}

1731

1732

void MSRepository::unlockRepo(RepoLockState state)

1733

{

1734

CSMutex *lock;

1735

enter_();

1736

lock = &myRepoLock[0];

1737

lock_(lock);

1738

1739

ASSERT(myRepoLockState & state);

1740

1741

myRepoLockState &= ~state;

1742

if (myRepoLockState == REPO_UNLOCKED) {

1743

myRepoXLock = false;

1744

signalCompactor();

1745

}

1746

unlock_(lock);

1747

1748

exit_();

1749

}

1750

1751

// Repositories are not removed from the pool when

1752

// scheduled for backup so the REPO_BACKUP flag is

1753

// not handled here.

1754

void MSRepository::returnToPool()

1755

{

1756

CSMutex *lock;

1757

enter_();

1758

lock = &myRepoLock[0];

1759

lock_(lock);

1760

this->myRepoLockState &= ~(REPO_COMPACTING | REPO_WRITE);

1761

if ( this->myRepoLockState == REPO_UNLOCKED) {

1762

myRepoXLock = false;

1763

signalCompactor();

1764

}

1765

unlock_(lock);

1766

1767

this->release();

1768

exit_();

1769

}

1770

1771

void MSRepository::backupCompleted()

1772

{

1773

CSMutex *lock;

1774

enter_();

1775

lock = &myRepoLock[0];

1776

lock_(lock);

1777

1778

1779

this->myRepoLockState &= ~REPO_BACKUP;

1780

if ( this->myRepoLockState == REPO_UNLOCKED) {

1781

myRepoXLock = false;

1782

signalCompactor();

1783

}

1784

1785

unlock_(lock);

1786

exit_();

1787

}

1788

1789

bool MSRepository::lockedForBackup() { return ((myRepoLockState & REPO_BACKUP) == REPO_BACKUP);}

1790

1791

u_int MSRepository::initBackup()

1792

{

1793

CSMutex *lock;

1794

u_int state;

1795

enter_();

1796

1797

lock = &myRepoLock[0];

1798

lock_(lock);

1799

state = this->myRepoLockState;

1800

this->myRepoLockState |= REPO_BACKUP;

1801

if (this->myRepoLockState == REPO_BACKUP)

1802

this->myRepoXLock = true;

1803

1804

unlock_(lock);

1805

return_(state);

1806

}

1807

1808

MSRepoFile *MSRepository::getRepoFile()

1809

{

1810

MSRepoFile *file;

1811

1812

if ((file = iFilePool)) {

1813

iFilePool = file->nextFile;

1814

file->nextFile = NULL;

1815

file->isFileInUse = true;

1816

file->retain();

1817

}

1818

return file;

1819

}

1820

1821

void MSRepository::addRepoFile(MSRepoFile *file)

1822

{

1823

iPoolFiles.addFront(file);

1824

}

1825

1826

void MSRepository::removeRepoFile(MSRepoFile *file)

1827

{

1828

iPoolFiles.remove(file);

1829

}

1830

1831

void MSRepository::returnRepoFile(MSRepoFile *file)

1832

{

1833

file->isFileInUse = false;

1834

file->nextFile = iFilePool;

1835

iFilePool = file;

1836

}

1837

1838

bool MSRepository::removeRepoFilesNotInUse()

1839

{

1840

MSRepoFile *file, *curr_file;

1841

1842

iFilePool = NULL;

1843

/* Remove all files that are not in use: */

1844

if ((file = (MSRepoFile *) iPoolFiles.getBack())) {

1845

do {

1846

curr_file = file;

1847

file = (MSRepoFile *) file->getNextLink();

1848

if (!curr_file->isFileInUse)

1849

iPoolFiles.remove(curr_file);

1850

} while (file);

1851

}

1852

return iPoolFiles.getSize() == 0;

1853

}

1854

1855

off_t MSRepository::getRepoFileSize()

1856

{

1857

return myRepoFileSize;

1858

}

1859

1860

size_t MSRepository::getRepoHeadSize()

1861

{

1862

return myRepoHeadSize;

1863

}

1864

1865

size_t MSRepository::getRepoBlobHeadSize()

1866

{

1867

return myRepoBlobHeadSize;

1868

}

1869

1870

CSMutex *MSRepository::getRepoLock(off_t offset)

1871

{

1872

return &myRepoLock[offset % CS_REPO_REC_LOCK_COUNT];

1873

}

1874

1875

u_int MSRepository::getRepoID()

1876

{

1877

return myRepoID;

1878

}

1879

1880

u_int MSRepository::getGarbageLevel()

1881

{

1882

if (myRepoFileSize <= myRepoHeadSize)

1883

return 0;

1884

return myGarbageCount * 100 / (myRepoFileSize - myRepoHeadSize);

1885

}

1886

1887

CSPath *MSRepository::getRepoFilePath()

1888

{

1889

char file_name[120];

1890

1891

cs_strcpy(120, file_name, "bs-repository");

1892

cs_add_dir_char(120, file_name);

1893

cs_strcat(120, file_name, "repo-");

1894

cs_strcat(120, file_name, myRepoID);

1895

cs_strcat(120, file_name, ".bs");

1896

1897

if (myRepoDatabase && myRepoDatabase->myDatabasePath) {

1898

return CSPath::newPath(RETAIN(myRepoDatabase->myDatabasePath), file_name);

1899

}

1900

return NULL;

1901

}

1902

Older »