Add AWS S3 protocol support to NetRocks plugin

.cirrus.yml

@@ -2,7 +2,7 @@ freebsd_test_task:
     freebsd_instance:
         image_family: freebsd-14-0
     name: cirrus/FreeBSD
-    install_script: pkg install -y cmake git pkgconf bash wx32-gtk3 libssh openssl libnfs neon libarchive samba416 uchardet libxml2
+    install_script: pkg install -y cmake git pkgconf bash wx32-gtk3 libssh openssl libnfs neon libarchive samba416 uchardet libxml2 aws-sdk-cpp
     script: mkdir build && cd build && cmake .. && make -j $(getconf _NPROCESSORS_ONLN) && ./install/far2l --help
 
 macos_test_task:

.travis.yml

@@ -5,6 +5,10 @@ matrix:
   include:
     - os: osx
       osx_image: xcode12
+      before_install:
+        - git clone https://github.com/microsoft/vcpkg.git
+        - ./vcpkg/bootstrap-vcpkg.sh
+        - ./vcpkg/vcpkg install aws-sdk-cpp
     - os: linux
       dist: xenial
       sudo: required
@@ -26,8 +30,12 @@ matrix:
             - libarchive-dev
             - ninja-build
       env: CXX=g++-5
+      before_install:
+        - git clone https://github.com/microsoft/vcpkg.git
+        - ./vcpkg/bootstrap-vcpkg.sh
+        - ./vcpkg/vcpkg install aws-sdk-cpp
       install:
-            - if [ "$CXX" = "g++" ]; then export CXX="g++-5" CC="gcc-5"; fi
+        - if [ "$CXX" = "g++" ]; then export CXX="g++-5" CC="gcc-5"; fi
     - os: linux
       dist: xenial
       sudo: required
@@ -49,8 +57,12 @@ matrix:
             - libarchive-dev
             - ninja-build
       env: CXX=g++-6
+      before_install:
+        - git clone https://github.com/microsoft/vcpkg.git
+        - ./vcpkg/bootstrap-vcpkg.sh
+        - ./vcpkg/vcpkg install aws-sdk-cpp
       install:
-            - if [ "$CXX" = "g++" ]; then export CXX="g++-6" CC="gcc-6"; fi
+        - if [ "$CXX" = "g++" ]; then export CXX="g++-6" CC="gcc-6"; fi
     - os: linux
       dist: xenial
       sudo: required
@@ -73,8 +85,12 @@ matrix:
             - libarchive-dev
             - ninja-build
       env: CXX=clang++-3.6
+      before_install:
+        - git clone https://github.com/microsoft/vcpkg.git
+        - ./vcpkg/bootstrap-vcpkg.sh
+        - ./vcpkg/vcpkg install aws-sdk-cpp
       install:
-            - if [ "$CXX" = "clang++" ]; then export CXX="clang++-3.6" CC="clang-3.6"; fi
+        - if [ "$CXX" = "clang++" ]; then export CXX="clang++-3.6" CC="clang-3.6"; fi
     - os: linux
       dist: xenial
       sudo: required
@@ -98,8 +114,12 @@ matrix:
             - zlib1g-dev
             - ninja-build
       env: CXX=clang++-3.7
+      before_install:
+        - git clone https://github.com/microsoft/vcpkg.git
+        - ./vcpkg/bootstrap-vcpkg.sh
+        - ./vcpkg/vcpkg install aws-sdk-cpp
       install:
-            - if [ "$CXX" = "clang++" ]; then export CXX="clang++-3.7" CC="clang-3.7"; fi
+        - if [ "$CXX" = "clang++" ]; then export CXX="clang++-3.7" CC="clang-3.7"; fi
 
 before_install:
   - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then brew unlink python@2 ; fi
@@ -112,5 +132,6 @@ before_install:
 script:
   - mkdir build
   - cd build
-  - cmake -G Ninja .. && ninja -v
+  - cmake -G Ninja .. -DCMAKE_TOOLCHAIN_FILE=../vcpkg/scripts/buildsystems/vcpkg.cmake
+  - ninja -v
   - cd install && zip -r ../far2l-${TRAVIS_COMMIT}.zip * && cd -

Brewfile

@@ -10,3 +10,5 @@ brew "uchardet"
 brew "wget"
 brew "wxwidgets"
 brew "libxml2"
+brew "libxml2"
+brew "aws-sdk-cpp"

CMakeLists.txt

@@ -454,6 +454,13 @@ if (NOT DEFINED NETROCKS OR NETROCKS)
         message(WARNING "${ColorRed}libneon not found, NetRocks will not have WebDav protocol support. Install libneon*-dev if you want WebDav protocol available in NetRocks.${ColorNormal}")
     endif(LIBNEON_FOUND)
 
+    find_package(AWSSDK QUIET COMPONENTS s3)
+    if(AWSSDK_FOUND)
+        message(STATUS "aws-sdk-cpp found -> enjoy AWS s3 support in NetRocks")
+    else()
+        message(WARNING "${ColorRed}aws-sdk-cpp not found, NetRocks will not have AWS S3 protocol support. Install aws-sdk-cpp if you want AWS S3 protocol available in NetRocks.${ColorNormal}")
+    endif(LIBNEON_FOUND)
+
     add_subdirectory (NetRocks)
 else()
     message(STATUS "${ColorRed}NETROCKS plugin disabled due to NETROCKS=${NETROCKS}${ColorNormal}")

NetRocks/CMakeLists.txt

@@ -23,6 +23,7 @@ src/UI/Settings/ExtraSiteSettings.cpp
 src/UI/Settings/ProxySettings.cpp
 src/UI/Settings/ConfigureProtocolSFTPSCP.cpp
 src/UI/Settings/ConfigureProtocolFTP.cpp
+src/UI/Settings/ConfigureProtocolAWS.cpp
 src/UI/Settings/ConfigureProtocolSMB.cpp
 src/UI/Settings/ConfigureProtocolNFS.cpp
 src/UI/Settings/ConfigureProtocolWebDAV.cpp
@@ -173,6 +174,26 @@ if (LIBNEON_FOUND AND ((NOT DEFINED NR_WEBDAV) OR NR_WEBDAV))
         SUFFIX ".broker")
 endif ()
 
+if (AWSSDK_FOUND AND ((NOT DEFINED NR_AWS) OR NR_AWS))
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DHAVE_AWS")
+    add_executable (NetRocks-AWS
+        ${PROTOCOL_SOURCES}
+        src/Protocol/AWS/AWSFile.cpp
+        src/Protocol/AWS/AWSFileWriter.cpp
+        src/Protocol/AWS/AWSFileReader.cpp
+        src/Protocol/AWS/S3Repository.cpp
+        src/Protocol/AWS/ProtocolAWS.cpp
+    )
+    target_include_directories(NetRocks-AWS PRIVATE src ${AWSSDK_INCLUDE_DIR})
+    target_link_libraries(NetRocks-AWS utils ${AWSSDK_LINK_LIBRARIES})
+
+    set_target_properties(NetRocks-AWS
+        PROPERTIES
+        RUNTIME_OUTPUT_DIRECTORY "${INSTALL_DIR}/Plugins/NetRocks/plug"
+        PREFIX ""
+        SUFFIX ".broker")
+endif ()
+
     add_executable (NetRocks-FTP
         ${PROTOCOL_SOURCES}
         src/Protocol/FTP/ProtocolFTP.cpp

NetRocks/configs/plug/bel.lng

@@ -316,3 +316,13 @@
 "Не выкарыстоўваць праксіфікатар"
 "Выкарыстоўваны &праксіфікатар:"
 "&Рэдагаваць канфігурацыю праксіфікатара"
+
+"Ховішча AWS S3"
+"Карыстальніцкі агент"
+"Рэгіён"
+"Выкарыстоўваць праксы"
+"Хост праксы"
+"Порт праксы"
+"Пракса з аўтэнтыфікацыяй"
+"Імя карыстальніка праксы"
+"Пароль праксы"

NetRocks/configs/plug/eng.lng

@@ -316,3 +316,13 @@
 "Not using proxifier"
 "&Proxifier kind to use:"
 "&Edit proxifier configuration"
+
+"AWS S3 Storage"
+"User agent"
+"Region"
+"Use proxy"
+"Proxy host"
+"Proxy port"
+"Auth proxy"
+"Proxy username"
+"Proxy password"

NetRocks/configs/plug/rus.lng

@@ -316,3 +316,13 @@
 "Не использовать проксификатор"
 "Используемый &проксификатор:"
 "&Редактировать конфигурацию проксификатора"
+
+"Хранилище AWS S3"
+"Пользовательский агент"
+"Регион"
+"Использовать прокси"
+"Хост прокси"
+"Порт прокси"
+"Прокси с аутентификацией"
+"Имя пользователя прокси"
+"Пароль прокси"

NetRocks/src/Protocol/AWS/AWSFile.cpp

@@ -0,0 +1,21 @@
+#include "AWSFile.h"
+
+AWSFile::AWSFile(std::string name, bool isFile): name(name), isFile(isFile) 
+{
+    modified.tv_sec = 0;
+    modified.tv_nsec = 0;
+}
+AWSFile::AWSFile(std::string name, bool isFile, const Aws::Utils::DateTime &date, long long size): name(name), isFile(isFile) 
+{
+    modified.tv_sec = static_cast<time_t>(date.Seconds());
+    modified.tv_nsec = 0;
+    this->size = size;
+}
+
+void AWSFile::UpdateModification(const Aws::Utils::DateTime &date)
+{
+    auto t = static_cast<time_t>(date.Seconds());
+    if (t > modified.tv_sec) {
+        modified.tv_sec = t;
+    }
+}

NetRocks/src/Protocol/AWS/AWSFile.h

@@ -0,0 +1,17 @@
+#pragma once
+#include <string>
+#include <aws/core/Aws.h>
+
+class AWSFile
+{
+public:
+	std::string name;
+	bool isFile;
+	timespec modified;
+	long long size = 0;
+
+	AWSFile(std::string name, bool isFile);
+	AWSFile(std::string name, bool isFile, const Aws::Utils::DateTime &date, long long size);
+
+    void UpdateModification(const Aws::Utils::DateTime &date);
+};

NetRocks/src/Protocol/AWS/AWSFileReader.cpp

@@ -0,0 +1,55 @@
+#include "AWSFileReader.h"
+#include <aws/s3/model/GetObjectRequest.h>
+
+AWSFileReader::AWSFileReader(
+        std::shared_ptr<Aws::S3::S3Client> client, 
+        const std::string& backet, 
+        const std::string& key, 
+        unsigned long long position, 
+        unsigned long long size
+    ): _client(client), _backet(backet), _key(key), _position(position), _size(size)
+{
+    floatBufferSize = std::min((size - position) / 10, MAX_BUFFER);
+    if (floatBufferSize == 0) {
+        floatBufferSize = std::min(size - position, MAX_BUFFER);
+    }
+}
+
+size_t AWSFileReader::Read(void *buf, size_t len)
+{
+    if (buffer.empty()) {
+        if (_position + floatBufferSize > _size) {
+            Download(_backet, _key, _position, _size - _position);
+        } else {
+            Download(_backet, _key, _position, floatBufferSize);
+        }
+    }
+    size_t bytesRead = std::min(len, buffer.size());
+    std::memcpy(buf, buffer.data(), bytesRead);
+    buffer.erase(buffer.begin(), buffer.begin() + bytesRead);
+
+    _position += bytesRead;
+    return bytesRead;
+}
+
+size_t AWSFileReader::Download(std::string backet, std::string key, size_t offset, size_t len) 
+{
+    Aws::S3::Model::GetObjectRequest request;
+    request.SetBucket(backet);
+    request.SetKey(key);
+    Aws::String range = "bytes=" + std::to_string(offset) + "-" + std::to_string(offset + len - 1);
+    request.SetRange(range);
+
+    auto outcome = _client->GetObject(request);
+    if (!outcome.IsSuccess())
+    {
+        throw ProtocolError("Failed to get file: " + outcome.GetError().GetMessage());
+    }
+
+    auto &stream = outcome.GetResult().GetBody();
+    size_t currentSize = buffer.size();
+    buffer.resize(currentSize + len);
+    stream.read(buffer.data() + currentSize, len);
+
+    return stream.gcount();
+}

NetRocks/src/Protocol/AWS/AWSFileReader.h

@@ -0,0 +1,32 @@
+#pragma once
+
+#include "../Protocol.h"
+#include <aws/s3/S3Client.h>
+
+
+class AWSFileReader : public IFileReader
+{
+private:
+    std::shared_ptr<Aws::S3::S3Client> _client;
+	std::string _backet;
+	std::string _key;
+	unsigned long long _position;
+	unsigned long long _size;
+	std::vector<char> buffer;
+	static constexpr unsigned long long MAX_BUFFER = 10 * 1024 * 1024;
+	unsigned long long floatBufferSize;
+
+    size_t Download(std::string backet, std::string key, size_t offset, size_t len);
+
+public:
+	AWSFileReader(
+        std::shared_ptr<Aws::S3::S3Client> client, 
+        const std::string& backet, 
+        const std::string& key, 
+        unsigned long long position, 
+        unsigned long long size
+    );
+
+	virtual size_t Read(void *buf, size_t len);
+};
+