diff --git a/src/main/config.cpp b/src/main/config.cpp index 50b7b4299e..ebc3ee0e74 100644 --- a/src/main/config.cpp +++ b/src/main/config.cpp @@ -31,7 +31,6 @@ import status; import options; import command_statement; import infinity_exception; -import virtual_storage_type; namespace infinity { diff --git a/src/main/config.cppm b/src/main/config.cppm index 89e76b4d53..2b6ba4f70d 100644 --- a/src/main/config.cppm +++ b/src/main/config.cppm @@ -20,7 +20,7 @@ import third_party; import options; import status; import command_statement; -import virtual_storage_type; +import virtual_store; namespace infinity { diff --git a/src/storage/buffer/file_worker/bmp_index_file_worker.cpp b/src/storage/buffer/file_worker/bmp_index_file_worker.cpp index 13253f908c..eb9547a0aa 100644 --- a/src/storage/buffer/file_worker/bmp_index_file_worker.cpp +++ b/src/storage/buffer/file_worker/bmp_index_file_worker.cpp @@ -28,7 +28,6 @@ import bmp_alg; import abstract_bmp; import virtual_store; import persistence_manager; -import abstract_file_handle; import local_file_handle; namespace infinity { diff --git a/src/storage/io/abstract_file_handle.cpp b/src/storage/io/abstract_file_handle.cpp deleted file mode 100644 index ad79db9a9f..0000000000 --- a/src/storage/io/abstract_file_handle.cpp +++ /dev/null @@ -1,29 +0,0 @@ -// Copyright(C) 2024 InfiniFlow, Inc. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -module; - -module abstract_file_handle; - -import status; -import virtual_storage; - -namespace infinity { - -AbstractFileHandle::AbstractFileHandle(VirtualStorage *storage_system, StorageType storage_type) - : storage_system_(storage_system), storage_type_(storage_type) {} - -AbstractFileHandle::~AbstractFileHandle() = default; - -} // namespace infinity diff --git a/src/storage/io/abstract_file_handle.cppm b/src/storage/io/abstract_file_handle.cppm deleted file mode 100644 index 07c81a1c73..0000000000 --- a/src/storage/io/abstract_file_handle.cppm +++ /dev/null @@ -1,56 +0,0 @@ -// Copyright(C) 2024 InfiniFlow, Inc. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -module; - -export module abstract_file_handle; - -import stl; -import virtual_storage_type; -import status; - -namespace infinity { - -class VirtualStorage; - -export enum class FileAccessMode { kWrite, kRead, kMmapRead, kInvalid }; - -export class AbstractFileHandle { -public: - explicit AbstractFileHandle(VirtualStorage *storage_system, StorageType storage_type); - virtual ~AbstractFileHandle() = 0; - virtual Status Open(const String &path, FileAccessMode access_mode) = 0; - virtual Status Close() = 0; - virtual Status Append(const void *buffer, u64 nbytes) = 0; - virtual Status Append(const String &buffer, u64 nbytes) = 0; - virtual Tuple Read(void *buffer, u64 nbytes) = 0; - virtual Tuple Read(String &buffer, u64 nbytes) = 0; - virtual Status Seek(u64 nbytes) = 0; - virtual Status Download(const String& url, const String& path) = 0; // Download from url to path - virtual Status Upload(const String& path, const String& url) = 0; // Upload from path to url - virtual SizeT FileSize() = 0; - virtual Tuple MmapRead(const String &name) = 0; - virtual Status Unmmap(const String &name) = 0; - virtual Status Sync() = 0; - -protected: - VirtualStorage *storage_system_{}; - StorageType storage_type_{StorageType::kLocal}; - atomic_bool open_{false}; - String path_; - FileAccessMode access_mode_; - atomic_bool sync_{false}; -}; - -} // namespace infinity diff --git a/src/storage/io/file_handle/local_file.cpp b/src/storage/io/file_handle/local_file.cpp deleted file mode 100644 index 0f7b629bb1..0000000000 --- a/src/storage/io/file_handle/local_file.cpp +++ /dev/null @@ -1,182 +0,0 @@ -// Copyright(C) 2024 InfiniFlow, Inc. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -module; - -#include -#include -#include -#include -#include - -module local_file; - -import third_party; -import status; -import virtual_storage; -import infinity_exception; -import logger; - -namespace infinity { - -LocalFile::LocalFile(VirtualStorage *storage_system) : AbstractFileHandle(storage_system, StorageType::kLocal) {} - -LocalFile::~LocalFile() = default; - -Status LocalFile::Open(const String &path, FileAccessMode access_mode) { - if(!path_.empty()) { - return Status::FileIsOpen(path); - } - path_ = path; - access_mode_ = access_mode; - switch (access_mode_) { - case FileAccessMode::kRead: { - fd_ = open(path.c_str(), O_RDONLY, 0666); - break; - } - case FileAccessMode::kWrite: { - fd_ = open(path.c_str(), O_RDWR | O_CREAT, 0666); - break; - } - case FileAccessMode::kMmapRead: { - UnrecoverableError("Unsupported now."); - break; - } - case FileAccessMode::kInvalid: { - break; - } - } - if(fd_ == -1) { - String error_message = fmt::format("File open failed: {}", strerror(errno)); - return Status::IOError(error_message); - } - open_ = true; - return Status::OK(); -} - -Status LocalFile::Close() { - if(access_mode_ == FileAccessMode::kWrite) { - if(!sync_) { - Status status = Sync(); - if(!status.ok()) { - return status; - } - } - } - if(open_) { - close(fd_); - fd_ = -1; - open_ = false; - path_.clear(); - sync_ = false; - access_mode_ = FileAccessMode::kInvalid; - } - return Status::OK(); -} - -Status LocalFile::Append(const void *buffer, u64 nbytes) { - if(!open_ or access_mode_ != FileAccessMode::kWrite) { - String error_message = fmt::format("File: {} isn't open.", path_); - UnrecoverableError(error_message); - } - i64 written = 0; - while (written < (i64)nbytes) { - i64 write_count = write(fd_, (char*)buffer + written, nbytes - written); - if (write_count == -1) { - String error_message = fmt::format("Can't write file: {}: {}. fd: {}", path_, strerror(errno), fd_); - UnrecoverableError(error_message); - } - written += write_count; - } - return Status::OK(); -} - -Status LocalFile::Append(const String &buffer, u64 nbytes) { - return Append(buffer.data(), nbytes); -} - -Tuple LocalFile::Read(void *buffer, u64 nbytes) { - if(!open_) { - String error_message = fmt::format("File: {} isn't open.", path_); - UnrecoverableError(error_message); - } - i64 read_n = 0; - while (read_n < (i64)nbytes) { - SizeT a = nbytes - read_n; - i64 read_count = read(fd_, (char *)buffer + read_n, a); - if (read_count == 0) { - break; - } - if (read_count == -1) { - String error_message = fmt::format("Can't read file: {}: {}", path_, strerror(errno)); - UnrecoverableError(error_message); - } - read_n += read_count; - } - return {read_n, Status::OK()}; -} - -Tuple LocalFile::Read(String &buffer, u64 nbytes) { - i64 read_n = 0; - while (read_n < (i64)nbytes) { - SizeT a = nbytes - read_n; - i64 read_count = read(fd_, buffer.data() + read_n, a); - if (read_count == 0) { - break; - } - if (read_count == -1) { - String error_message = fmt::format("Can't read file: {}: {}", path_, strerror(errno)); - UnrecoverableError(error_message); - } - read_n += read_count; - } - return {read_n, Status::OK()}; -} - -Status LocalFile::Seek(u64 nbytes) { - if ((off_t)-1 == lseek(fd_, nbytes, SEEK_SET)) { - String error_message = fmt::format("Can't seek file: {}: {}", path_, strerror(errno)); - UnrecoverableError(error_message); - } - return Status::OK(); -} - -Status LocalFile::Download(const String &url, const String &path) { return Status::OK(); } - -Status LocalFile::Upload(const String &path, const String &url) { return Status::OK(); } - -SizeT LocalFile::FileSize() { - struct stat s {}; - if (fstat(fd_, &s) == -1) { - return -1; - } - return s.st_size; -} - -Tuple LocalFile::MmapRead(const String &name) { return {nullptr, 0, Status::OK()}; } - -Status LocalFile::Unmmap(const String &name) { return Status::OK(); } - -Status LocalFile::Sync() { - if(access_mode_ != FileAccessMode::kWrite) { - return Status::InvalidCommand("Non-write access mode, shouldn't call Sync()"); - } - if(!sync_) { - sync_ = true; - fsync(fd_); - } - return Status::OK(); -} - -} // namespace infinity diff --git a/src/storage/io/file_handle/local_file.cppm b/src/storage/io/file_handle/local_file.cppm deleted file mode 100644 index 78ed7afe1c..0000000000 --- a/src/storage/io/file_handle/local_file.cppm +++ /dev/null @@ -1,50 +0,0 @@ -// Copyright(C) 2024 InfiniFlow, Inc. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -module; - -export module local_file; - -import stl; -import virtual_storage_type; -import status; -import abstract_file_handle; - -namespace infinity { - -class VirtualStorage; - -export class LocalFile final : public AbstractFileHandle { -public: - explicit LocalFile(VirtualStorage *storage_system); - ~LocalFile() final; - Status Open(const String &path, FileAccessMode access_mode) final; - Status Close() final; - Status Append(const void *buffer, u64 nbytes) final; - Status Append(const String &buffer, u64 nbytes) final; - Tuple Read(void *buffer, u64 nbytes) final; - Tuple Read(String &buffer, u64 nbytes) final; - Status Seek(u64 nbytes) final; - Status Download(const String &url, const String &path) final; - Status Upload(const String &path, const String &url) final; - SizeT FileSize() final; - Tuple MmapRead(const String &name) final; - Status Unmmap(const String &name) final; - Status Sync() final; - -private: - i32 fd_{-1}; -}; - -} // namespace infinity \ No newline at end of file diff --git a/src/storage/io/file_handle/object_file.cpp b/src/storage/io/file_handle/object_file.cpp deleted file mode 100644 index 44ec9fd54a..0000000000 --- a/src/storage/io/file_handle/object_file.cpp +++ /dev/null @@ -1,54 +0,0 @@ -// Copyright(C) 2024 InfiniFlow, Inc. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -module; - -module object_file; - -import status; -import virtual_storage; - -namespace infinity { - -ObjectFile::ObjectFile(VirtualStorage *storage_system, StorageType type) : AbstractFileHandle(storage_system, type) {} - -ObjectFile::~ObjectFile() = default; - -Status ObjectFile::Open(const String &path, FileAccessMode access_mode) { return Status::OK(); } - -Status ObjectFile::Close() { return Status::OK(); } - -Status ObjectFile::Append(const void *buffer, u64 nbytes) { return Status::OK(); } - -Status ObjectFile::Append(const String &buffer, u64 nbytes) { return Status::OK(); } - -Tuple ObjectFile::Read(void *buffer, u64 nbytes) { return {0, Status::OK()}; } - -Tuple ObjectFile::Read(String &buffer, u64 nbytes) { return {0, Status::OK()}; } - -Status ObjectFile::Seek(u64 nbytes) { return Status::OK(); } - -Status ObjectFile::Download(const String &url, const String &path) { return Status::OK(); } - -Status ObjectFile::Upload(const String &path, const String &url) { return Status::OK(); } - -SizeT ObjectFile::FileSize() { return 0; } - -Tuple ObjectFile::MmapRead(const String &name) { return {nullptr, 0, Status::OK()}; } - -Status ObjectFile::Unmmap(const String &name) { return Status::OK(); } - -Status ObjectFile::Sync() { return Status::OK(); } - -} // namespace infinity diff --git a/src/storage/io/file_handle/object_file.cppm b/src/storage/io/file_handle/object_file.cppm deleted file mode 100644 index 75ea23b8a9..0000000000 --- a/src/storage/io/file_handle/object_file.cppm +++ /dev/null @@ -1,47 +0,0 @@ -// Copyright(C) 2024 InfiniFlow, Inc. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -module; - -export module object_file; - -import stl; -import virtual_storage_type; -import status; -import abstract_file_handle; - -namespace infinity { - -class VirtualStorage; - -export class ObjectFile : public AbstractFileHandle { -public: - ObjectFile(VirtualStorage *storage_system, StorageType type); - ~ObjectFile() override; - Status Open(const String &path, FileAccessMode access_mode) override; - Status Close() override; - Status Append(const void *buffer, u64 nbytes) override; - Status Append(const String &buffer, u64 nbytes) override; - Tuple Read(void *buffer, u64 nbytes) override; - Tuple Read(String &buffer, u64 nbytes) override; - Status Seek(u64 nbytes) override; - Status Download(const String& url, const String& path) override; - Status Upload(const String& path, const String& url) override; - SizeT FileSize() override; - Tuple MmapRead(const String &name) override; - Status Unmmap(const String &name) override; - Status Sync() override; -}; - -} \ No newline at end of file diff --git a/src/storage/io/file_handle/minio_file.cpp b/src/storage/io/minio_file.cpp similarity index 99% rename from src/storage/io/file_handle/minio_file.cpp rename to src/storage/io/minio_file.cpp index dad1e1262b..bdb2b102ec 100644 --- a/src/storage/io/file_handle/minio_file.cpp +++ b/src/storage/io/minio_file.cpp @@ -11,7 +11,7 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. - +#if 0 module; #include @@ -308,4 +308,5 @@ Status MinioFile::Sync() { return Status::OK(); } -} \ No newline at end of file +} +#endif \ No newline at end of file diff --git a/src/storage/io/file_handle/minio_file.cppm b/src/storage/io/minio_file.cppm similarity index 97% rename from src/storage/io/file_handle/minio_file.cppm rename to src/storage/io/minio_file.cppm index 345d547fd4..eb32863b50 100644 --- a/src/storage/io/file_handle/minio_file.cppm +++ b/src/storage/io/minio_file.cppm @@ -12,12 +12,13 @@ // See the License for the specific language governing permissions and // limitations under the License. + module; export module minio_file; +#if 0 import stl; -import virtual_storage_type; import status; import object_file; import abstract_file_handle; @@ -55,4 +56,5 @@ private: static HashMap mapped_files_; }; -} // namespace infinity \ No newline at end of file +} // namespace infinity +#endif \ No newline at end of file diff --git a/src/storage/io/virtual_storage.cpp b/src/storage/io/virtual_storage.cpp deleted file mode 100644 index 6e9ebd6140..0000000000 --- a/src/storage/io/virtual_storage.cpp +++ /dev/null @@ -1,347 +0,0 @@ -// Copyright(C) 2024 InfiniFlow, Inc. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -module; - -#include - -module virtual_storage; - -import stl; -import third_party; -import virtual_storage_type; -import logger; -import local_file; -import minio_file; -import infinity_exception; -import default_values; - -namespace infinity { - -Status VirtualStorage::Init(StorageType storage_type, Map &config) { - // Init remote filesystem and local disk cache - storage_type_ = storage_type; - switch (storage_type) { - case StorageType::kLocal: { - if (!config.empty()) { - return Status::InvalidConfig("Local filesystem won't access any config"); - } - break; - } - case StorageType::kMinio: { - auto iter = config.find("url"); - if (iter == config.end()) { - return Status::InvalidConfig("Missing MINIO 'URL'"); - } - String url = iter->second; - - iter = config.find("access_key"); - if (iter == config.end()) { - return Status::InvalidConfig("Missing MINIO 'access_key'"); - } - String access_key = iter->second; - - iter = config.find("secret_key"); - if (iter == config.end()) { - return Status::InvalidConfig("Missing MINIO 'secret_key'"); - } - String secret_key = iter->second; - - iter = config.find("enable_https"); - if (iter == config.end()) { - return Status::InvalidConfig("Missing MINIO 'enable_https'"); - } - String enable_https_str = iter->second; - bool enable_https{false}; - if (enable_https_str == "true") { - enable_https = true; - } else if (enable_https_str == "false") { - enable_https = false; - } else { - return Status::InvalidConfig(fmt::format("Invalid MINIO 'enable_https' value: {}", enable_https_str)); - } - - iter = config.find("disk_cache_dir"); - if (iter == config.end()) { - return Status::InvalidConfig("Missing MINIO 'cache_dir'"); - } - SharedPtr cache_dir_ptr = MakeShared(iter->second); - - iter = config.find("disk_cache_limit"); - if (iter == config.end()) { - return Status::InvalidConfig("Missing MINIO 'cache_dir'"); - } - String disk_cache_limit_str = iter->second; - SizeT disk_cache_limit = std::stoull(disk_cache_limit_str); - - iter = config.find("disk_cache_lru_count"); - if (iter == config.end()) { - return Status::InvalidConfig("Missing MINIO 'cache_dir'"); - } - String disk_cache_lru_count_str = iter->second; - SizeT disk_cache_lru_count = std::stoull(disk_cache_lru_count_str); - - minio_base_url_ = MakeUnique(url, enable_https); - minio_provider_ = MakeUnique(access_key, secret_key); - minio_client_ = MakeUnique(*minio_base_url_, minio_provider_.get()); - local_disk_cache_ = MakeUnique(disk_cache_limit, cache_dir_ptr, disk_cache_lru_count); - break; - } - default: { - return Status::NotSupport(fmt::format("{} isn't support in virtual filesystem", ToString(storage_type))); - } - } - return Status::OK(); -} - -Status VirtualStorage::UnInit() { - minio_client_.reset(); - minio_provider_.reset(); - minio_base_url_.reset(); - local_disk_cache_.reset(); - return Status::OK(); -} - -Tuple, Status> VirtualStorage::BuildFileHandle() { - // Open the file according to the path and access_mode - switch (storage_type_) { - case StorageType::kLocal: { - UniquePtr local_file = MakeUnique(this); - return {std::move(local_file), Status::OK()}; - } - case StorageType::kMinio: { - UniquePtr minio_file = MakeUnique(this); - return {std::move(minio_file), Status::OK()}; - } - default: { - Status status = Status::NotSupport(fmt::format("{} isn't support in virtual filesystem", ToString(storage_type_))); - LOG_ERROR(status.message()); - return {nullptr, status}; - } - } - return {nullptr, Status::OK()}; -} - -LocalDiskCache *VirtualStorage::GetLocalDiskCache() const { return local_disk_cache_.get(); } - -Status VirtualStorage::DeleteFile(const String &file_name) { - switch (storage_type_) { - case StorageType::kLocal: { - return DeleteFileLocal(file_name); - } - default: { - Status status = Status::NotSupport(fmt::format("{} isn't support in virtual filesystem", ToString(storage_type_))); - LOG_ERROR(status.message()); - return status; - } - } - return Status::OK(); -} - -bool VirtualStorage::Exists(const String &path) { - switch (storage_type_) { - case StorageType::kLocal: { - return ExistsLocal(path); - } - default: { - Status status = Status::NotSupport(fmt::format("{} isn't support in virtual filesystem", ToString(storage_type_))); - LOG_ERROR(status.message()); - return false; - } - } - return false; -} - -Tuple, Status> VirtualStorage::ListDirectory(const String &path) { - Vector result; - return {result, Status::OK()}; -} - -bool VirtualStorage::IsRegularFile(const String &path) { return false; } - -// For local disk filesystem, such as temp file, disk cache and WAL -bool VirtualStorage::ExistsLocal(const String &path) { - if (!std::filesystem::path(path).is_absolute()) { - String error_message = fmt::format("{} isn't absolute path.", path); - UnrecoverableError(error_message); - } - std::error_code error_code; - Path p{path}; - bool is_exists = std::filesystem::exists(p, error_code); - if (error_code.value() == 0) { - return is_exists; - } else { - String error_message = fmt::format("{} exists exception: {}", path, strerror(errno)); - UnrecoverableError(error_message); - } - return false; -} - -Status VirtualStorage::DeleteFileLocal(const String &file_name) { - if (!std::filesystem::path(file_name).is_absolute()) { - String error_message = fmt::format("{} isn't absolute path.", file_name); - UnrecoverableError(error_message); - } - std::error_code error_code; - Path p{file_name}; - bool is_deleted = std::filesystem::remove(p, error_code); - if (error_code.value() == 0) { - if (!is_deleted) { - String error_message = fmt::format("Failed to delete file: {}: {}", file_name, strerror(errno)); - LOG_WARN(error_message); - Status status = Status::IOError(error_message); - return status; - } - } else { - String error_message = fmt::format("Delete file {} exception: {}", file_name, strerror(errno)); - UnrecoverableError(error_message); - } - return Status::OK(); -} - -Status VirtualStorage::MakeDirectoryLocal(const String &path) { - if (!std::filesystem::path(path).is_absolute()) { - String error_message = fmt::format("{} isn't absolute path.", path); - UnrecoverableError(error_message); - } - std::error_code error_code; - Path p{path}; - std::filesystem::create_directories(p, error_code); - if (error_code.value() != 0) { - String error_message = fmt::format("{} create exception: {}", path, strerror(errno)); - UnrecoverableError(error_message); - } - return Status::OK(); -} - -Status VirtualStorage::RemoveDirectoryLocal(const String &path) { - if (!std::filesystem::path(path).is_absolute()) { - String error_message = fmt::format("{} isn't absolute path.", path); - UnrecoverableError(error_message); - } - std::error_code error_code; - Path p{path}; - std::filesystem::remove_all(p, error_code); - if (error_code.value() != 0) { - String error_message = fmt::format("Delete directory {} exception: {}", path, error_code.message()); - UnrecoverableError(error_message); - } - return Status::OK(); -} - -Status VirtualStorage::CleanupDirectoryLocal(const String &path) { - if (!std::filesystem::path(path).is_absolute()) { - String error_message = fmt::format("{} isn't absolute path.", path); - UnrecoverableError(error_message); - } - std::error_code error_code; - Path p{path}; - if (!std::filesystem::exists(p)) { - std::filesystem::create_directories(p, error_code); - if (error_code.value() != 0) { - String error_message = fmt::format("CleanupDirectory create {} exception: {}", path, error_code.message()); - UnrecoverableError(error_message); - } - return Status::OK(); - } - try { - std::ranges::for_each(std::filesystem::directory_iterator{path}, [&](const auto &dir_entry) { std::filesystem::remove_all(dir_entry); }); - } catch (const std::filesystem::filesystem_error &e) { - String error_message = fmt::format("CleanupDirectory cleanup {} exception: {}", path, e.what()); - UnrecoverableError(error_message); - } - return Status::OK(); -} - -Status VirtualStorage::RenameLocal(const String &old_path, const String &new_path) { - if (!std::filesystem::path(old_path).is_absolute()) { - String error_message = fmt::format("{} isn't absolute path.", old_path); - UnrecoverableError(error_message); - } - if (!std::filesystem::path(new_path).is_absolute()) { - String error_message = fmt::format("{} isn't absolute path.", new_path); - UnrecoverableError(error_message); - } - if (rename(old_path.c_str(), new_path.c_str()) != 0) { - String error_message = fmt::format("Can't rename file: {}, {}", old_path, strerror(errno)); - UnrecoverableError(error_message); - } - return Status::OK(); -} - -Status VirtualStorage::TruncateLocal(const String &file_name, SizeT new_length) { - if (!std::filesystem::path(file_name).is_absolute()) { - String error_message = fmt::format("{} isn't absolute path.", file_name); - UnrecoverableError(error_message); - } - std::error_code error_code; - std::filesystem::resize_file(file_name, new_length, error_code); - if (error_code.value() != 0) { - String error_message = fmt::format("Failed to truncate {} to size {}", file_name, strerror(errno)); - UnrecoverableError(error_message); - } - return Status::OK(); -} - -Status VirtualStorage::MergeLocal(const String &dst_path, const String &src_path) { - if (!std::filesystem::path(dst_path).is_absolute()) { - String error_message = fmt::format("{} isn't absolute path.", dst_path); - UnrecoverableError(error_message); - } - if (!std::filesystem::path(src_path).is_absolute()) { - String error_message = fmt::format("{} isn't absolute path.", src_path); - UnrecoverableError(error_message); - } - Path dst{dst_path}; - Path src{src_path}; - std::ifstream srcFile(src, std::ios::binary); - if (!srcFile.is_open()) { - String error_message = fmt::format("Failed to open source file {}", src_path); - UnrecoverableError(error_message); - return Status::OK(); - } - std::ofstream dstFile(dst, std::ios::binary | std::ios::app); - if (!dstFile.is_open()) { - String error_message = fmt::format("Failed to open destination file {}", dst_path); - UnrecoverableError(error_message); - return Status::OK(); - } - char buffer[DEFAULT_READ_BUFFER_SIZE]; - while (srcFile.read(buffer, DEFAULT_READ_BUFFER_SIZE)) { - dstFile.write(buffer, srcFile.gcount()); - } - dstFile.write(buffer, srcFile.gcount()); - srcFile.close(); - dstFile.close(); - return Status::OK(); -} - -Tuple>, Status> VirtualStorage::ListDirectoryLocal(const String &path) { - if (!std::filesystem::path(path).is_absolute()) { - String error_message = fmt::format("{} isn't absolute path.", path); - UnrecoverableError(error_message); - } - Path dir_path(path); - if (!is_directory(dir_path)) { - String error_message = fmt::format("{} isn't a directory", path); - UnrecoverableError(error_message); - } - - Vector> file_array; - std::ranges::for_each(std::filesystem::directory_iterator{path}, - [&](const auto &dir_entry) { file_array.emplace_back(MakeShared(dir_entry)); }); - return {file_array, Status::OK()}; -} - -} // namespace infinity diff --git a/src/storage/io/virtual_storage.cppm b/src/storage/io/virtual_storage.cppm deleted file mode 100644 index fc397957b1..0000000000 --- a/src/storage/io/virtual_storage.cppm +++ /dev/null @@ -1,76 +0,0 @@ -// Copyright(C) 2024 InfiniFlow, Inc. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -module; - -export module virtual_storage; - -import stl; -import status; -import third_party; -import virtual_storage_type; -import abstract_file_handle; - -namespace infinity { - -export class LocalDiskCache { -public: - LocalDiskCache(u64 disk_capacity_limit, SharedPtr cache_dir, SizeT lru_count) : cache_dir_(std::move(cache_dir)) {} - - ~LocalDiskCache() = default; - - String GetCacheDir() const { return *cache_dir_; } - -private: - SharedPtr cache_dir_{}; -}; - -export class VirtualStorage { - // Factory class to generate different type of file handle -public: - Status Init(StorageType storage_type, Map &config); - Status UnInit(); - Tuple, Status> BuildFileHandle(); - LocalDiskCache *GetLocalDiskCache() const; - minio::s3::Client* GetMinioClient() const { - return minio_client_.get(); - } - - Status DeleteFile(const String& path); - - bool Exists(const String& path); - Tuple, Status> ListDirectory(const String& path); - bool IsRegularFile(const String& path); - - // For local disk filesystem, such as temp file, disk cache and WAL - static bool ExistsLocal(const String& path); - static Status DeleteFileLocal(const String& path); - static Status MakeDirectoryLocal(const String& path); - static Status RemoveDirectoryLocal(const String& path); - static Status CleanupDirectoryLocal(const String& path); - static Status RenameLocal(const String& old_path, const String& new_path); - static Status TruncateLocal(const String& file_name, SizeT new_length); - static Status MergeLocal(const String& dst_file, const String& src_file); - static Tuple>, Status> ListDirectoryLocal(const String& path); -private: - StorageType storage_type_{StorageType::kLocal}; - UniquePtr local_disk_cache_{}; - - // Using by minio - UniquePtr minio_base_url_{}; - UniquePtr minio_provider_{}; - UniquePtr minio_client_{}; -}; - -} // namespace infinity diff --git a/src/storage/io/virtual_storage_type.cpp b/src/storage/io/virtual_storage_type.cpp deleted file mode 100644 index 8cdfbd4dda..0000000000 --- a/src/storage/io/virtual_storage_type.cpp +++ /dev/null @@ -1,106 +0,0 @@ -// Copyright(C) 2024 InfiniFlow, Inc. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -module; - -#include - -module virtual_storage_type ; - -import stl; - -namespace infinity { - -StorageType String2StorageType(const String &storage_type) { - if (storage_type == "local") { - return StorageType::kLocal; - } - - if (storage_type == "minio") { - return StorageType::kMinio; - } - - if (storage_type == "aws_s3") { - return StorageType::kLocal; - } - - if (storage_type == "azure_blob") { - return StorageType::kLocal; - } - - if (storage_type == "gcs") { - return StorageType::kLocal; - } - - if (storage_type == "oss") { - return StorageType::kLocal; - } - - if (storage_type == "cos") { - return StorageType::kLocal; - } - if (storage_type == "obs") { - return StorageType::kLocal; - } - - if (storage_type == "hdfs") { - return StorageType::kLocal; - } - - if (storage_type == "nfs") { - return StorageType::kLocal; - } - - return StorageType::kInvalid; -} - - -String ToString(StorageType storage_type) { - switch (storage_type) { - case StorageType::kLocal: { - return "local"; - } - case StorageType::kMinio: { - return "minio"; - } - case StorageType::kAwsS3: { - return "aws s3"; - } - case StorageType::kAzureBlob: { - return "azure blob"; - } - case StorageType::kGCS: { - return "google cloud storage"; - } - case StorageType::kOSS: { - return "aliyun object storage service"; - } - case StorageType::kCOS: { - return "tencent cloud object storage"; - } - case StorageType::kOBS: { - return "huawei object storage service"; - } - case StorageType::kHDFS: { - return "hadoop file system"; - } - case StorageType::kNFS: { - return "network file system"; - } - default: { - return "invalid"; - } - } -} -} diff --git a/src/storage/io/virtual_storage_type.cppm b/src/storage/io/virtual_storage_type.cppm deleted file mode 100644 index 64f2216790..0000000000 --- a/src/storage/io/virtual_storage_type.cppm +++ /dev/null @@ -1,41 +0,0 @@ -// Copyright(C) 2024 InfiniFlow, Inc. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -module; - -export module virtual_storage_type; - -import stl; - -namespace infinity { - -export enum class StorageType { - kInvalid, - kLocal, - kMinio, - kAwsS3, - kAzureBlob, // Azure object store - kGCS, // Google cloud storage - kOSS, // Aliyun OSS - kCOS, // Tencent object store - kOBS, // Huawei object store - kHDFS, - kNFS, -}; - -export StorageType String2StorageType(const String &storage_type); - -export String ToString(StorageType storage_type); - -} // namespace infinity \ No newline at end of file diff --git a/src/storage/io/virtual_store.cpp b/src/storage/io/virtual_store.cpp index d189aaf015..b262ba4ae5 100644 --- a/src/storage/io/virtual_store.cpp +++ b/src/storage/io/virtual_store.cpp @@ -27,7 +27,6 @@ module virtual_store; import stl; import third_party; -import virtual_storage_type; import logger; import infinity_exception; import default_values; @@ -35,6 +34,89 @@ import stream_reader; namespace infinity { + +StorageType String2StorageType(const String &storage_type) { + if (storage_type == "local") { + return StorageType::kLocal; + } + + if (storage_type == "minio") { + return StorageType::kMinio; + } + + if (storage_type == "aws_s3") { + return StorageType::kLocal; + } + + if (storage_type == "azure_blob") { + return StorageType::kLocal; + } + + if (storage_type == "gcs") { + return StorageType::kLocal; + } + + if (storage_type == "oss") { + return StorageType::kLocal; + } + + if (storage_type == "cos") { + return StorageType::kLocal; + } + if (storage_type == "obs") { + return StorageType::kLocal; + } + + if (storage_type == "hdfs") { + return StorageType::kLocal; + } + + if (storage_type == "nfs") { + return StorageType::kLocal; + } + + return StorageType::kInvalid; +} + + +String ToString(StorageType storage_type) { + switch (storage_type) { + case StorageType::kLocal: { + return "local"; + } + case StorageType::kMinio: { + return "minio"; + } + case StorageType::kAwsS3: { + return "aws s3"; + } + case StorageType::kAzureBlob: { + return "azure blob"; + } + case StorageType::kGCS: { + return "google cloud storage"; + } + case StorageType::kOSS: { + return "aliyun object storage service"; + } + case StorageType::kCOS: { + return "tencent cloud object storage"; + } + case StorageType::kOBS: { + return "huawei object storage service"; + } + case StorageType::kHDFS: { + return "hadoop file system"; + } + case StorageType::kNFS: { + return "network file system"; + } + default: { + return "invalid"; + } + } +} + Status RemoteStore::Init(StorageType storage_type, Map &config) { // Init remote filesystem and local disk cache storage_type_ = storage_type; diff --git a/src/storage/io/virtual_store.cppm b/src/storage/io/virtual_store.cppm index 88d1351082..0ad00552ed 100644 --- a/src/storage/io/virtual_store.cppm +++ b/src/storage/io/virtual_store.cppm @@ -20,11 +20,28 @@ import stl; import status; import third_party; import local_file_handle; -import virtual_storage_type; import stream_reader; namespace infinity { +export enum class StorageType { + kInvalid, + kLocal, + kMinio, + kAwsS3, + kAzureBlob, // Azure object store + kGCS, // Google cloud storage + kOSS, // Aliyun OSS + kCOS, // Tencent object store + kOBS, // Huawei object store + kHDFS, + kNFS, +}; + +export StorageType String2StorageType(const String &storage_type); + +export String ToString(StorageType storage_type); + export struct MmapInfo { u8 *data_ptr_{}; SizeT data_len_{}; diff --git a/src/storage/storage.cpp b/src/storage/storage.cpp index 44fa27558a..ce8a9b31d0 100644 --- a/src/storage/storage.cpp +++ b/src/storage/storage.cpp @@ -50,7 +50,6 @@ import cleanup_scanner; import persistence_manager; import extra_ddl_info; import virtual_store; -import virtual_storage_type; namespace infinity { diff --git a/src/unit_test/main/config.cpp b/src/unit_test/main/config.cpp index 8da253b546..817daf591c 100644 --- a/src/unit_test/main/config.cpp +++ b/src/unit_test/main/config.cpp @@ -21,7 +21,7 @@ import infinity_exception; import third_party; import status; import compilation_config; -import virtual_storage_type; +import virtual_store; using namespace infinity; class ConfigTest : public BaseTest {}; diff --git a/src/unit_test/storage/io/file_handle/local_file.cpp b/src/unit_test/storage/io/file_handle/local_file.cpp index 9570ad1799..912a32b3a7 100644 --- a/src/unit_test/storage/io/file_handle/local_file.cpp +++ b/src/unit_test/storage/io/file_handle/local_file.cpp @@ -17,7 +17,6 @@ import base_test; import stl; import virtual_store; -import virtual_storage_type; import local_file_handle; using namespace infinity; diff --git a/src/unit_test/storage/io/file_handle/minio_file.cpp b/src/unit_test/storage/io/file_handle/minio_file.cpp index 7441d953f8..4e9adb405b 100644 --- a/src/unit_test/storage/io/file_handle/minio_file.cpp +++ b/src/unit_test/storage/io/file_handle/minio_file.cpp @@ -18,7 +18,6 @@ import stl; import status; import infinity_context; import compilation_config; -import virtual_storage_type; import virtual_store; import minio_file; diff --git a/src/unit_test/storage/meta/entry/block_version.cpp b/src/unit_test/storage/meta/entry/block_version.cpp index 3539d15856..2b09e67ef8 100644 --- a/src/unit_test/storage/meta/entry/block_version.cpp +++ b/src/unit_test/storage/meta/entry/block_version.cpp @@ -24,7 +24,6 @@ import third_party; import infinity_context; import block_version; import virtual_store; -import virtual_storage_type; import buffer_manager; import version_file_worker; import column_vector; diff --git a/src/unit_test/storage/persistence/persistence_manager.cpp b/src/unit_test/storage/persistence/persistence_manager.cpp index 5f95636e7b..bb3bfbbc6c 100644 --- a/src/unit_test/storage/persistence/persistence_manager.cpp +++ b/src/unit_test/storage/persistence/persistence_manager.cpp @@ -3,7 +3,6 @@ import base_test; import stl; import persistence_manager; import virtual_store; -import virtual_storage_type; import third_party; import persist_result_handler; import local_file_handle;