diff --git a/db/c.cc b/db/c.cc index dd97ed8ae21..b101540ffa1 100644 --- a/db/c.cc +++ b/db/c.cc @@ -1858,10 +1858,6 @@ extern ROCKSDB_LIBRARY_API void rocksdb_approximate_sizes_cf_with_flags( delete[] ranges; } -void DEPRECATED_rocksdb_delete_file(rocksdb_t* db, const char* name) { - db->rep->DEPRECATED_DeleteFile(name); -} - const rocksdb_livefiles_t* rocksdb_livefiles(rocksdb_t* db) { rocksdb_livefiles_t* result = new rocksdb_livefiles_t; db->rep->GetLiveFilesMetaData(&result->rep); diff --git a/db/db_impl/db_impl.cc b/db/db_impl/db_impl.cc index 48aa47c27f1..97fa3ccbcf6 100644 --- a/db/db_impl/db_impl.cc +++ b/db/db_impl/db_impl.cc @@ -4883,108 +4883,6 @@ Status DBImpl::GetUpdatesSince( return wal_manager_.GetUpdatesSince(seq, iter, read_options, versions_.get()); } -Status DBImpl::DEPRECATED_DeleteFile(std::string name) { - // TODO: plumb Env::IOActivity, Env::IOPriority - const ReadOptions read_options; - const WriteOptions write_options; - - uint64_t number; - FileType type; - WalFileType log_type; - if (!ParseFileName(name, &number, &type, &log_type) || - (type != kTableFile && type != kWalFile)) { - ROCKS_LOG_ERROR(immutable_db_options_.info_log, "DeleteFile %s failed.\n", - name.c_str()); - return Status::InvalidArgument("Invalid file name"); - } - - if (type == kWalFile) { - // Only allow deleting archived log files - if (log_type != kArchivedLogFile) { - ROCKS_LOG_ERROR(immutable_db_options_.info_log, - "DeleteFile %s failed - not archived log.\n", - name.c_str()); - return Status::NotSupported("Delete only supported for archived logs"); - } - Status status = wal_manager_.DeleteFile(name, number); - if (!status.ok()) { - ROCKS_LOG_ERROR(immutable_db_options_.info_log, - "DeleteFile %s failed -- %s.\n", name.c_str(), - status.ToString().c_str()); - } - return status; - } - - Status status; - int level; - FileMetaData* metadata; - ColumnFamilyData* cfd; - VersionEdit edit; - JobContext job_context(next_job_id_.fetch_add(1), true); - { - InstrumentedMutexLock l(&mutex_); - status = versions_->GetMetadataForFile(number, &level, &metadata, &cfd); - if (!status.ok()) { - ROCKS_LOG_WARN(immutable_db_options_.info_log, - "DeleteFile %s failed. File not found\n", name.c_str()); - job_context.Clean(); - return Status::InvalidArgument("File not found"); - } - assert(level < cfd->NumberLevels()); - - // If the file is being compacted no need to delete. - if (metadata->being_compacted) { - ROCKS_LOG_INFO(immutable_db_options_.info_log, - "DeleteFile %s Skipped. File about to be compacted\n", - name.c_str()); - job_context.Clean(); - return Status::OK(); - } - - // Only the files in the last level can be deleted externally. - // This is to make sure that any deletion tombstones are not - // lost. Check that the level passed is the last level. - auto* vstoreage = cfd->current()->storage_info(); - for (int i = level + 1; i < cfd->NumberLevels(); i++) { - if (vstoreage->NumLevelFiles(i) != 0) { - ROCKS_LOG_WARN(immutable_db_options_.info_log, - "DeleteFile %s FAILED. File not in last level\n", - name.c_str()); - job_context.Clean(); - return Status::InvalidArgument("File not in last level"); - } - } - // if level == 0, it has to be the oldest file - if (level == 0 && - vstoreage->LevelFiles(0).back()->fd.GetNumber() != number) { - ROCKS_LOG_WARN(immutable_db_options_.info_log, - "DeleteFile %s failed ---" - " target file in level 0 must be the oldest.", - name.c_str()); - job_context.Clean(); - return Status::InvalidArgument("File in level 0, but not oldest"); - } - edit.SetColumnFamily(cfd->GetID()); - edit.DeleteFile(level, number); - status = versions_->LogAndApply(cfd, read_options, write_options, &edit, - &mutex_, directories_.GetDbDir()); - if (status.ok()) { - InstallSuperVersionAndScheduleWork( - cfd, job_context.superversion_contexts.data()); - } - FindObsoleteFiles(&job_context, false); - } // lock released here - - LogFlush(immutable_db_options_.info_log); - // remove files outside the db-lock - if (job_context.HaveSomethingToDelete()) { - // Call PurgeObsoleteFiles() without holding mutex. - PurgeObsoleteFiles(job_context); - } - job_context.Clean(); - return status; -} - Status DBImpl::DeleteFilesInRanges(ColumnFamilyHandle* column_family, const RangePtr* ranges, size_t n, bool include_end) { diff --git a/db/db_impl/db_impl.h b/db/db_impl/db_impl.h index 1e1af5eb780..bb2ff6eee95 100644 --- a/db/db_impl/db_impl.h +++ b/db/db_impl/db_impl.h @@ -535,7 +535,6 @@ class DBImpl : public DB { SequenceNumber seq_number, std::unique_ptr* iter, const TransactionLogIterator::ReadOptions& read_options = TransactionLogIterator::ReadOptions()) override; - Status DEPRECATED_DeleteFile(std::string name) override; Status DeleteFilesInRanges(ColumnFamilyHandle* column_family, const RangePtr* ranges, size_t n, bool include_end = true); diff --git a/db/db_test.cc b/db/db_test.cc index 003f3c55058..6bb900bc811 100644 --- a/db/db_test.cc +++ b/db/db_test.cc @@ -3409,10 +3409,6 @@ class ModelDB : public DB { return Status::NotSupported(); } - Status DEPRECATED_DeleteFile(std::string /*name*/) override { - return Status::OK(); - } - Status GetUpdatesSince( ROCKSDB_NAMESPACE::SequenceNumber, std::unique_ptr*, @@ -5319,7 +5315,7 @@ TEST_F(DBTest, DynamicLevelCompressionPerLevel) { options.compression_per_level[0] = kNoCompression; // No compression for the Ln whre L0 is compacted to options.compression_per_level[1] = kNoCompression; - // Snpapy compression for Ln+1 + // Snappy compression for Ln+1 options.compression_per_level[2] = kSnappyCompression; OnFileDeletionListener* listener = new OnFileDeletionListener(); @@ -5373,7 +5369,10 @@ TEST_F(DBTest, DynamicLevelCompressionPerLevel) { db_->GetColumnFamilyMetaData(&cf_meta); for (const auto& file : cf_meta.levels[4].files) { listener->SetExpectedFileName(dbname_ + file.name); - ASSERT_OK(dbfull()->DEPRECATED_DeleteFile(file.name)); + Slice start(file.smallestkey), limit(file.largestkey); + const RangePtr ranges(&start, &limit); + EXPECT_OK(dbfull()->DeleteFilesInRanges(dbfull()->DefaultColumnFamily(), + &ranges, true /* include_end */)); } listener->VerifyMatchedCount(cf_meta.levels[4].files.size()); diff --git a/db/deletefile_test.cc b/db/deletefile_test.cc index a915924ad3b..17ce6b80fb4 100644 --- a/db/deletefile_test.cc +++ b/db/deletefile_test.cc @@ -135,57 +135,6 @@ class DeleteFileTest : public DBTestBase { } }; -TEST_F(DeleteFileTest, AddKeysAndQueryLevels) { - Options options = CurrentOptions(); - SetOptions(&options); - Destroy(options); - options.create_if_missing = true; - Reopen(options); - - CreateTwoLevels(); - std::vector metadata; - db_->GetLiveFilesMetaData(&metadata); - - std::string level1file; - int level1keycount = 0; - std::string level2file; - int level2keycount = 0; - int level1index = 0; - int level2index = 1; - - ASSERT_EQ((int)metadata.size(), 2); - if (metadata[0].level == 2) { - level1index = 1; - level2index = 0; - } - - level1file = metadata[level1index].name; - int startkey = atoi(metadata[level1index].smallestkey.c_str()); - int endkey = atoi(metadata[level1index].largestkey.c_str()); - level1keycount = (endkey - startkey + 1); - level2file = metadata[level2index].name; - startkey = atoi(metadata[level2index].smallestkey.c_str()); - endkey = atoi(metadata[level2index].largestkey.c_str()); - level2keycount = (endkey - startkey + 1); - - // COntrolled setup. Levels 1 and 2 should both have 50K files. - // This is a little fragile as it depends on the current - // compaction heuristics. - ASSERT_EQ(level1keycount, 50000); - ASSERT_EQ(level2keycount, 50000); - - Status status = db_->DEPRECATED_DeleteFile("0.sst"); - ASSERT_TRUE(status.IsInvalidArgument()); - - // intermediate level files cannot be deleted. - status = db_->DEPRECATED_DeleteFile(level1file); - ASSERT_TRUE(status.IsInvalidArgument()); - - // Lowest level file deletion should succeed. - status = db_->DEPRECATED_DeleteFile(level2file); - ASSERT_OK(status); -} - TEST_F(DeleteFileTest, PurgeObsoleteFilesTest) { Options options = CurrentOptions(); SetOptions(&options); @@ -496,145 +445,6 @@ TEST_F(DeleteFileTest, BackgroundPurgeTestMultipleJobs) { CheckFileTypeCounts(dbname_, 0, 1, 1); } -TEST_F(DeleteFileTest, DeleteFileWithIterator) { - Options options = CurrentOptions(); - SetOptions(&options); - Destroy(options); - options.create_if_missing = true; - Reopen(options); - - CreateTwoLevels(); - ReadOptions read_options; - Iterator* it = db_->NewIterator(read_options); - ASSERT_OK(it->status()); - std::vector metadata; - db_->GetLiveFilesMetaData(&metadata); - - std::string level2file; - - ASSERT_EQ(metadata.size(), static_cast(2)); - if (metadata[0].level == 1) { - level2file = metadata[1].name; - } else { - level2file = metadata[0].name; - } - - Status status = db_->DEPRECATED_DeleteFile(level2file); - fprintf(stdout, "Deletion status %s: %s\n", level2file.c_str(), - status.ToString().c_str()); - ASSERT_OK(status); - it->SeekToFirst(); - int numKeysIterated = 0; - while (it->Valid()) { - numKeysIterated++; - it->Next(); - } - ASSERT_EQ(numKeysIterated, 50000); - delete it; -} - -TEST_F(DeleteFileTest, DeleteLogFiles) { - Options options = CurrentOptions(); - SetOptions(&options); - Destroy(options); - options.create_if_missing = true; - Reopen(options); - - AddKeys(10, 0); - VectorLogPtr logfiles; - ASSERT_OK(db_->GetSortedWalFiles(logfiles)); - ASSERT_GT(logfiles.size(), 0UL); - // Take the last log file which is expected to be alive and try to delete it - // Should not succeed because live logs are not allowed to be deleted - std::unique_ptr alive_log = std::move(logfiles.back()); - ASSERT_EQ(alive_log->Type(), kAliveLogFile); - ASSERT_OK(env_->FileExists(wal_dir_ + "/" + alive_log->PathName())); - fprintf(stdout, "Deleting alive log file %s\n", - alive_log->PathName().c_str()); - ASSERT_NOK(db_->DEPRECATED_DeleteFile(alive_log->PathName())); - ASSERT_OK(env_->FileExists(wal_dir_ + "/" + alive_log->PathName())); - logfiles.clear(); - - // Call Flush to bring about a new working log file and add more keys - // Call Flush again to flush out memtable and move alive log to archived log - // and try to delete the archived log file - FlushOptions fopts; - ASSERT_OK(db_->Flush(fopts)); - AddKeys(10, 0); - ASSERT_OK(db_->Flush(fopts)); - ASSERT_OK(db_->GetSortedWalFiles(logfiles)); - ASSERT_GT(logfiles.size(), 0UL); - std::unique_ptr archived_log = std::move(logfiles.front()); - ASSERT_EQ(archived_log->Type(), kArchivedLogFile); - ASSERT_OK(env_->FileExists(wal_dir_ + "/" + archived_log->PathName())); - fprintf(stdout, "Deleting archived log file %s\n", - archived_log->PathName().c_str()); - ASSERT_OK(db_->DEPRECATED_DeleteFile(archived_log->PathName())); - ASSERT_TRUE( - env_->FileExists(wal_dir_ + "/" + archived_log->PathName()).IsNotFound()); -} - -TEST_F(DeleteFileTest, DeleteNonDefaultColumnFamily) { - Options options = CurrentOptions(); - SetOptions(&options); - Destroy(options); - options.create_if_missing = true; - Reopen(options); - CreateAndReopenWithCF({"new_cf"}, options); - - Random rnd(5); - for (int i = 0; i < 1000; ++i) { - ASSERT_OK(db_->Put(WriteOptions(), handles_[1], test::RandomKey(&rnd, 10), - test::RandomKey(&rnd, 10))); - } - ASSERT_OK(db_->Flush(FlushOptions(), handles_[1])); - for (int i = 0; i < 1000; ++i) { - ASSERT_OK(db_->Put(WriteOptions(), handles_[1], test::RandomKey(&rnd, 10), - test::RandomKey(&rnd, 10))); - } - ASSERT_OK(db_->Flush(FlushOptions(), handles_[1])); - - std::vector metadata; - db_->GetLiveFilesMetaData(&metadata); - ASSERT_EQ(2U, metadata.size()); - ASSERT_EQ("new_cf", metadata[0].column_family_name); - ASSERT_EQ("new_cf", metadata[1].column_family_name); - auto old_file = metadata[0].smallest_seqno < metadata[1].smallest_seqno - ? metadata[0].name - : metadata[1].name; - auto new_file = metadata[0].smallest_seqno > metadata[1].smallest_seqno - ? metadata[0].name - : metadata[1].name; - ASSERT_TRUE(db_->DEPRECATED_DeleteFile(new_file).IsInvalidArgument()); - ASSERT_OK(db_->DEPRECATED_DeleteFile(old_file)); - - { - std::unique_ptr itr(db_->NewIterator(ReadOptions(), handles_[1])); - ASSERT_OK(itr->status()); - int count = 0; - for (itr->SeekToFirst(); itr->Valid(); itr->Next()) { - ASSERT_OK(itr->status()); - ++count; - } - ASSERT_OK(itr->status()); - ASSERT_EQ(count, 1000); - } - - Close(); - ReopenWithColumnFamilies({kDefaultColumnFamilyName, "new_cf"}, options); - - { - std::unique_ptr itr(db_->NewIterator(ReadOptions(), handles_[1])); - int count = 0; - for (itr->SeekToFirst(); itr->Valid(); itr->Next()) { - ASSERT_OK(itr->status()); - ++count; - } - ASSERT_OK(itr->status()); - ASSERT_EQ(count, 1000); - } -} - } // namespace ROCKSDB_NAMESPACE int main(int argc, char** argv) { diff --git a/include/rocksdb/c.h b/include/rocksdb/c.h index b188b5a1f52..f2616ea3e7f 100644 --- a/include/rocksdb/c.h +++ b/include/rocksdb/c.h @@ -692,9 +692,6 @@ extern ROCKSDB_LIBRARY_API void rocksdb_compact_range_cf_opt( rocksdb_compactoptions_t* opt, const char* start_key, size_t start_key_len, const char* limit_key, size_t limit_key_len); -extern ROCKSDB_LIBRARY_API void DEPRECATED_rocksdb_delete_file( - rocksdb_t* db, const char* name); - extern ROCKSDB_LIBRARY_API const rocksdb_livefiles_t* rocksdb_livefiles( rocksdb_t* db); diff --git a/include/rocksdb/db.h b/include/rocksdb/db.h index a647e3045a4..41bbb68af7a 100644 --- a/include/rocksdb/db.h +++ b/include/rocksdb/db.h @@ -1836,19 +1836,6 @@ class DB { const TransactionLogIterator::ReadOptions& read_options = TransactionLogIterator::ReadOptions()) = 0; - // WARNING: This API is planned for removal in RocksDB 7.0 since it does not - // operate at the proper level of abstraction for a key-value store, and its - // contract/restrictions are poorly documented. For example, it returns non-OK - // `Status` for non-bottommost files and files undergoing compaction. Since we - // do not plan to maintain it, the contract will likely remain underspecified - // until its removal. Any user is encouraged to read the implementation - // carefully and migrate away from it when possible. - // - // Delete the file name from the db directory and update the internal state to - // reflect that. Supports deletion of sst and log files only. 'name' must be - // path relative to the db directory. eg. 000001.sst, /archive/000003.log - virtual Status DEPRECATED_DeleteFile(std::string name) = 0; - // Obtains a list of all live table (SST) files and how they fit into the // LSM-trees, such as column family, level, key range, etc. // This builds a de-normalized form of GetAllColumnFamilyMetaData(). diff --git a/include/rocksdb/utilities/stackable_db.h b/include/rocksdb/utilities/stackable_db.h index 0e604dea538..244989a6c98 100644 --- a/include/rocksdb/utilities/stackable_db.h +++ b/include/rocksdb/utilities/stackable_db.h @@ -518,17 +518,6 @@ class StackableDB : public DB { return db_->GetCreationTimeOfOldestFile(creation_time); } - // WARNING: This API is planned for removal in RocksDB 7.0 since it does not - // operate at the proper level of abstraction for a key-value store, and its - // contract/restrictions are poorly documented. For example, it returns non-OK - // `Status` for non-bottommost files and files undergoing compaction. Since we - // do not plan to maintain it, the contract will likely remain underspecified - // until its removal. Any user is encouraged to read the implementation - // carefully and migrate away from it when possible. - Status DEPRECATED_DeleteFile(std::string name) override { - return db_->DEPRECATED_DeleteFile(name); - } - Status GetDbIdentity(std::string& identity) const override { return db_->GetDbIdentity(identity); } diff --git a/java/rocksjni/rocksjni.cc b/java/rocksjni/rocksjni.cc index d3e71b380c7..9561b389366 100644 --- a/java/rocksjni/rocksjni.cc +++ b/java/rocksjni/rocksjni.cc @@ -3240,25 +3240,6 @@ jlong Java_org_rocksdb_RocksDB_getUpdatesSince(JNIEnv* env, jclass, return 0; } -/* - * Class: org_rocksdb_RocksDB - * Method: deprecated_deleteFile - * Signature: (JLjava/lang/String;)V - */ -void Java_org_rocksdb_RocksDB_deprecated_1deleteFile(JNIEnv* env, jclass, - jlong jdb_handle, - jstring jname) { - auto* db = reinterpret_cast(jdb_handle); - jboolean has_exception = JNI_FALSE; - std::string name = - ROCKSDB_NAMESPACE::JniUtil::copyStdString(env, jname, &has_exception); - if (has_exception == JNI_TRUE) { - // exception occurred - return; - } - db->DEPRECATED_DeleteFile(name); -} - /* * Class: org_rocksdb_RocksDB * Method: getLiveFilesMetaData diff --git a/java/src/main/java/org/rocksdb/RocksDB.java b/java/src/main/java/org/rocksdb/RocksDB.java index 0e4b917dc64..1ffb44b6a1b 100644 --- a/java/src/main/java/org/rocksdb/RocksDB.java +++ b/java/src/main/java/org/rocksdb/RocksDB.java @@ -4419,20 +4419,6 @@ public TransactionLogIterator getUpdatesSince(final long sequenceNumber) getUpdatesSince(nativeHandle_, sequenceNumber)); } - /** - * Delete the file name from the db directory and update the internal state to - * reflect that. Supports deletion of sst and log files only. 'name' must be - * path relative to the db directory. eg. 000001.sst, /archive/000003.log - * - * @param name the file name - * - * @throws RocksDBException if an error occurs whilst deleting the file - */ - @Deprecated - public void deprecated_deleteFile(final String name) throws RocksDBException { - deprecated_deleteFile(nativeHandle_, name); - } - /** * Gets a list of all table files metadata. * @@ -5055,9 +5041,6 @@ private static native String[] getLiveFiles(final long handle, final boolean flu private static native LogFile[] getSortedWalFiles(final long handle) throws RocksDBException; private static native long getUpdatesSince(final long handle, final long sequenceNumber) throws RocksDBException; - @Deprecated - private static native void deprecated_deleteFile(final long handle, final String name) - throws RocksDBException; private static native LiveFileMetaData[] getLiveFilesMetaData(final long handle); private static native ColumnFamilyMetaData getColumnFamilyMetaData( final long handle, final long columnFamilyHandle); diff --git a/java/src/test/java/org/rocksdb/EventListenerTest.java b/java/src/test/java/org/rocksdb/EventListenerTest.java index 0dfa9dc626f..03364ae6871 100644 --- a/java/src/test/java/org/rocksdb/EventListenerTest.java +++ b/java/src/test/java/org/rocksdb/EventListenerTest.java @@ -75,20 +75,49 @@ public void onFlushBegin(final RocksDB rocksDb, final FlushJobInfo flushJobInfo) void deleteTableFile(final AbstractEventListener el, final AtomicBoolean wasCbCalled) throws RocksDBException { - try (final Options opt = - new Options().setCreateIfMissing(true).setListeners(Collections.singletonList(el)); + final int KEY_SIZE = 20; + final int VALUE_SIZE = 1000; + final int FILE_SIZE = 64000; + final int NUM_FILES = 2; + + final int KEY_INTERVAL = 10000; + /* + * Intention of these options is to end up reliably with NUM_FILES files. + * we will be deleting using deleteFilesInRange. + * It is writing roughly number of keys that will fit in NUM_FILES files (target size). + * It is writing interleaved so that files from memory on L0 will overlap. + * Then compaction cleans everything, and we should end up with NUM_FILES files. + */ + try (final Options opt = new Options() + .setCreateIfMissing(true) + .setListeners(Collections.singletonList(el)) + .setCompressionType(CompressionType.NO_COMPRESSION) + .setTargetFileSizeBase(FILE_SIZE) + .setWriteBufferSize(FILE_SIZE / 2) + .setDisableAutoCompactions(true) + .setLevelCompactionDynamicLevelBytes(false); final RocksDB db = RocksDB.open(opt, dbFolder.getRoot().getAbsolutePath())) { - assertThat(db).isNotNull(); - final byte[] value = new byte[24]; - rand.nextBytes(value); - db.put("testKey".getBytes(), value); - final RocksDB.LiveFiles liveFiles = db.getLiveFiles(); - assertThat(liveFiles).isNotNull(); - assertThat(liveFiles.files).isNotNull(); - assertThat(liveFiles.files.isEmpty()).isFalse(); - db.deprecated_deleteFile(liveFiles.files.get(0)); - assertThat(wasCbCalled.get()).isTrue(); + final int records = FILE_SIZE / (KEY_SIZE + VALUE_SIZE); + + // fill database with key/value pairs + final byte[] value = new byte[VALUE_SIZE]; + int key_init = 0; + for (int o = 0; o < NUM_FILES; ++o) { + int int_key = key_init++; + for (int i = 0; i < records; ++i) { + int_key += KEY_INTERVAL; + rand.nextBytes(value); + + db.put(String.format("%020d", int_key).getBytes(), value); + } + } + try (final FlushOptions flushOptions = new FlushOptions().setWaitForFlush(true)) { + db.flush(flushOptions); + } + db.compactRange(); + db.deleteFilesInRanges(null, Arrays.asList(null, null), false /* includeEnd */); } + assertThat(wasCbCalled.get()).isTrue(); } @Test diff --git a/java/src/test/java/org/rocksdb/RocksDBTest.java b/java/src/test/java/org/rocksdb/RocksDBTest.java index 38eba6ca153..5a9c76fd8e3 100644 --- a/java/src/test/java/org/rocksdb/RocksDBTest.java +++ b/java/src/test/java/org/rocksdb/RocksDBTest.java @@ -1665,16 +1665,6 @@ public void getSortedWalFiles() throws RocksDBException { } } - @Test - public void deprecated_deleteFile() throws RocksDBException { - try (final Options options = new Options().setCreateIfMissing(true)) { - final String dbPath = dbFolder.getRoot().getAbsolutePath(); - try (final RocksDB db = RocksDB.open(options, dbPath)) { - db.deprecated_deleteFile("unknown"); - } - } - } - @Test public void getLiveFilesMetaData() throws RocksDBException { try (final Options options = new Options().setCreateIfMissing(true)) {