From 0d456645c76b1828b8d1edf6e8060949d4b5833f Mon Sep 17 00:00:00 2001 From: Kan Tang Date: Mon, 10 Sep 2018 15:12:50 +0800 Subject: [PATCH] CPPLite initial commit (#1) * CPPLite initial commit --- .gitignore | 372 +-- CMakeLists.txt | 177 ++ CMakeSettings.json | 58 + NOTICES.txt | 17 + README.md | 99 +- include/append_block_request_base.h | 29 + include/base64.h | 13 + include/blob/append_block_request.h | 42 + include/blob/blob_client.h | 463 ++++ include/blob/copy_blob_request.h | 43 + include/blob/create_block_blob_request.h | 103 + include/blob/create_container_request.h | 29 + include/blob/delete_blob_request.h | 41 + include/blob/delete_container_request.h | 21 + include/blob/download_blob_request.h | 54 + include/blob/get_blob_property_request.h | 29 + include/blob/get_block_list_request.h | 29 + include/blob/get_container_property_request.h | 23 + include/blob/get_page_ranges_request.h | 55 + include/blob/list_blobs_request.h | 120 + include/blob/list_containers_request.h | 53 + include/blob/put_block_list_request.h | 53 + include/blob/put_block_request.h | 50 + include/blob/put_page_request.h | 77 + include/common.h | 124 + include/constants.dat | 102 + include/constants.h | 9 + include/copy_blob_request_base.h | 26 + include/create_container_request_base.h | 35 + include/delete_blob_request_base.h | 32 + include/delete_container_request_base.h | 22 + include/executor.h | 207 ++ include/get_blob_property_request_base.h | 65 + include/get_blob_request_base.h | 42 + include/get_block_list_request_base.h | 46 + include/get_container_property_request_base.h | 46 + include/get_page_ranges_request_base.h | 40 + include/hash.h | 67 + include/http/libcurl_http_client.h | 275 ++ include/http_base.h | 69 + include/list_blobs_request_base.h | 108 + include/list_containers_request_base.h | 47 + include/put_blob_request_base.h | 50 + include/put_block_list_request_base.h | 47 + include/put_block_request_base.h | 27 + include/put_page_request_base.h | 39 + include/retry.h | 100 + include/storage_EXPORTS.h | 7 + include/storage_account.h | 43 + include/storage_credential.h | 76 + include/storage_errno.h | 23 + include/storage_outcome.h | 79 + include/storage_request_base.h | 36 + include/storage_stream.h | 149 ++ include/storage_url.h | 77 + include/tinyxml2.h | 2130 +++++++++++++++ include/tinyxml2_parser.h | 45 + include/todo/get_blob_metadata_request.h | 78 + include/todo/get_blob_properties_request.h | 76 + include/todo/query_entities_request.h | 87 + include/todo/set_blob_metadata_request.h | 82 + include/utility.h | 184 ++ include/xml_parser_base.h | 63 + include/xml_writer.h | 56 + sample/CMakeLists.txt | 4 + sample/Makefile | 3 + sample/sample.cpp | 106 + src/append_block_request_base.cpp | 41 + src/base64.cpp | 233 ++ src/blob/blob_client.cpp | 384 +++ src/blob/blob_client_wrapper.cpp | 882 +++++++ src/constants.cpp | 9 + src/copy_blob_request_base.cpp | 44 + src/create_container_request_base.cpp | 48 + src/delete_blob_request_base.cpp | 50 + src/delete_container_request_base.cpp | 34 + src/get_blob_property_request_base.cpp | 36 + src/get_blob_request_base.cpp | 44 + src/get_block_list_request_base.cpp | 49 + src/get_container_property_request_base.cpp | 35 + src/get_page_ranges_request_base.cpp | 38 + src/hash.cpp | 71 + src/http/libcurl_http_client.cpp | 88 + src/list_blobs_request_base.cpp | 115 + src/list_containers_request_base.cpp | 39 + src/put_blob_request_base.cpp | 79 + src/put_block_list_request_base.cpp | 60 + src/put_block_request_base.cpp | 36 + src/put_page_request_base.cpp | 54 + src/storage_account.cpp | 67 + src/storage_credential.cpp | 87 + src/storage_url.cpp | 117 + src/tinyxml2.cpp | 2344 +++++++++++++++++ src/tinyxml2_parser.cpp | 265 ++ src/utility.cpp | 110 + test/CMakeLists.txt | 28 + test/CMakeLists.txt.in | 15 + .../append_blob_integration_test.cpp | 92 + .../blob_container_integration_test.cpp | 218 ++ .../blob_general_integration_test.cpp | 250 ++ test/integration/blob_integration_base.cpp | 35 + test/integration/blob_integration_base.h | 11 + .../block_blob_integration_test.cpp | 347 +++ .../page_blob_integration_test.cpp | 254 ++ test/test_base.cpp | 165 ++ test/test_base.h | 26 + test/test_constants.cpp | 5 + test/test_constants.h | 5 + 108 files changed, 13667 insertions(+), 322 deletions(-) create mode 100644 CMakeLists.txt create mode 100644 CMakeSettings.json create mode 100644 NOTICES.txt create mode 100644 include/append_block_request_base.h create mode 100644 include/base64.h create mode 100644 include/blob/append_block_request.h create mode 100644 include/blob/blob_client.h create mode 100644 include/blob/copy_blob_request.h create mode 100644 include/blob/create_block_blob_request.h create mode 100644 include/blob/create_container_request.h create mode 100644 include/blob/delete_blob_request.h create mode 100644 include/blob/delete_container_request.h create mode 100644 include/blob/download_blob_request.h create mode 100644 include/blob/get_blob_property_request.h create mode 100644 include/blob/get_block_list_request.h create mode 100644 include/blob/get_container_property_request.h create mode 100644 include/blob/get_page_ranges_request.h create mode 100644 include/blob/list_blobs_request.h create mode 100644 include/blob/list_containers_request.h create mode 100644 include/blob/put_block_list_request.h create mode 100644 include/blob/put_block_request.h create mode 100644 include/blob/put_page_request.h create mode 100644 include/common.h create mode 100644 include/constants.dat create mode 100644 include/constants.h create mode 100644 include/copy_blob_request_base.h create mode 100644 include/create_container_request_base.h create mode 100644 include/delete_blob_request_base.h create mode 100644 include/delete_container_request_base.h create mode 100644 include/executor.h create mode 100644 include/get_blob_property_request_base.h create mode 100644 include/get_blob_request_base.h create mode 100644 include/get_block_list_request_base.h create mode 100644 include/get_container_property_request_base.h create mode 100644 include/get_page_ranges_request_base.h create mode 100644 include/hash.h create mode 100644 include/http/libcurl_http_client.h create mode 100644 include/http_base.h create mode 100644 include/list_blobs_request_base.h create mode 100644 include/list_containers_request_base.h create mode 100644 include/put_blob_request_base.h create mode 100644 include/put_block_list_request_base.h create mode 100644 include/put_block_request_base.h create mode 100644 include/put_page_request_base.h create mode 100644 include/retry.h create mode 100644 include/storage_EXPORTS.h create mode 100644 include/storage_account.h create mode 100644 include/storage_credential.h create mode 100644 include/storage_errno.h create mode 100644 include/storage_outcome.h create mode 100644 include/storage_request_base.h create mode 100644 include/storage_stream.h create mode 100644 include/storage_url.h create mode 100644 include/tinyxml2.h create mode 100644 include/tinyxml2_parser.h create mode 100644 include/todo/get_blob_metadata_request.h create mode 100644 include/todo/get_blob_properties_request.h create mode 100644 include/todo/query_entities_request.h create mode 100644 include/todo/set_blob_metadata_request.h create mode 100644 include/utility.h create mode 100644 include/xml_parser_base.h create mode 100644 include/xml_writer.h create mode 100644 sample/CMakeLists.txt create mode 100644 sample/Makefile create mode 100644 sample/sample.cpp create mode 100644 src/append_block_request_base.cpp create mode 100644 src/base64.cpp create mode 100644 src/blob/blob_client.cpp create mode 100644 src/blob/blob_client_wrapper.cpp create mode 100644 src/constants.cpp create mode 100644 src/copy_blob_request_base.cpp create mode 100644 src/create_container_request_base.cpp create mode 100644 src/delete_blob_request_base.cpp create mode 100644 src/delete_container_request_base.cpp create mode 100644 src/get_blob_property_request_base.cpp create mode 100644 src/get_blob_request_base.cpp create mode 100644 src/get_block_list_request_base.cpp create mode 100644 src/get_container_property_request_base.cpp create mode 100644 src/get_page_ranges_request_base.cpp create mode 100644 src/hash.cpp create mode 100644 src/http/libcurl_http_client.cpp create mode 100644 src/list_blobs_request_base.cpp create mode 100644 src/list_containers_request_base.cpp create mode 100644 src/put_blob_request_base.cpp create mode 100644 src/put_block_list_request_base.cpp create mode 100644 src/put_block_request_base.cpp create mode 100644 src/put_page_request_base.cpp create mode 100644 src/storage_account.cpp create mode 100644 src/storage_credential.cpp create mode 100644 src/storage_url.cpp create mode 100644 src/tinyxml2.cpp create mode 100644 src/tinyxml2_parser.cpp create mode 100644 src/utility.cpp create mode 100644 test/CMakeLists.txt create mode 100644 test/CMakeLists.txt.in create mode 100644 test/integration/append_blob_integration_test.cpp create mode 100644 test/integration/blob_container_integration_test.cpp create mode 100644 test/integration/blob_general_integration_test.cpp create mode 100644 test/integration/blob_integration_base.cpp create mode 100644 test/integration/blob_integration_base.h create mode 100644 test/integration/block_blob_integration_test.cpp create mode 100644 test/integration/page_blob_integration_test.cpp create mode 100644 test/test_base.cpp create mode 100644 test/test_base.h create mode 100644 test/test_constants.cpp create mode 100644 test/test_constants.h diff --git a/.gitignore b/.gitignore index 3e759b7..1935e46 100644 --- a/.gitignore +++ b/.gitignore @@ -1,330 +1,60 @@ -## Ignore Visual Studio temporary files, build results, and -## files generated by popular Visual Studio add-ons. -## -## Get latest from https://github.com/github/gitignore/blob/master/VisualStudio.gitignore +# Prerequisites +*.d -# User-specific files -*.suo -*.user -*.userosscache -*.sln.docstates - -# User-specific files (MonoDevelop/Xamarin Studio) -*.userprefs - -# Build results -[Dd]ebug/ -[Dd]ebugPublic/ -[Rr]elease/ -[Rr]eleases/ -x64/ -x86/ -bld/ -[Bb]in/ -[Oo]bj/ -[Ll]og/ - -# Visual Studio 2015/2017 cache/options directory -.vs/ -# Uncomment if you have tasks that create the project's static files in wwwroot -#wwwroot/ - -# Visual Studio 2017 auto generated files -Generated\ Files/ - -# MSTest test Results -[Tt]est[Rr]esult*/ -[Bb]uild[Ll]og.* - -# NUNIT -*.VisualState.xml -TestResult.xml - -# Build Results of an ATL Project -[Dd]ebugPS/ -[Rr]eleasePS/ -dlldata.c - -# Benchmark Results -BenchmarkDotNet.Artifacts/ - -# .NET Core -project.lock.json -project.fragment.lock.json -artifacts/ -**/Properties/launchSettings.json - -# StyleCop -StyleCopReport.xml - -# Files built by Visual Studio -*_i.c -*_p.c -*_i.h -*.ilk -*.meta +# Object files +*.o +*.ko *.obj -*.iobj -*.pch -*.pdb -*.ipdb -*.pgc -*.pgd -*.rsp -*.sbr -*.tlb -*.tli -*.tlh -*.tmp -*.tmp_proj -*.log -*.vspscc -*.vssscc -.builds -*.pidb -*.svclog -*.scc - -# Chutzpah Test files -_Chutzpah* - -# Visual C++ cache files -ipch/ -*.aps -*.ncb -*.opendb -*.opensdf -*.sdf -*.cachefile -*.VC.db -*.VC.VC.opendb - -# Visual Studio profiler -*.psess -*.vsp -*.vspx -*.sap - -# Visual Studio Trace Files -*.e2e - -# TFS 2012 Local Workspace -$tf/ - -# Guidance Automation Toolkit -*.gpState - -# ReSharper is a .NET coding add-in -_ReSharper*/ -*.[Rr]e[Ss]harper -*.DotSettings.user - -# JustCode is a .NET coding add-in -.JustCode - -# TeamCity is a build add-in -_TeamCity* - -# DotCover is a Code Coverage Tool -*.dotCover - -# AxoCover is a Code Coverage Tool -.axoCover/* -!.axoCover/settings.json - -# Visual Studio code coverage results -*.coverage -*.coveragexml - -# NCrunch -_NCrunch_* -.*crunch*.local.xml -nCrunchTemp_* - -# MightyMoose -*.mm.* -AutoTest.Net/ - -# Web workbench (sass) -.sass-cache/ - -# Installshield output folder -[Ee]xpress/ - -# DocProject is a documentation generator add-in -DocProject/buildhelp/ -DocProject/Help/*.HxT -DocProject/Help/*.HxC -DocProject/Help/*.hhc -DocProject/Help/*.hhk -DocProject/Help/*.hhp -DocProject/Help/Html2 -DocProject/Help/html - -# Click-Once directory -publish/ +*.elf -# Publish Web Output -*.[Pp]ublish.xml -*.azurePubxml -# Note: Comment the next line if you want to checkin your web deploy settings, -# but database connection strings (with potential passwords) will be unencrypted -*.pubxml -*.publishproj - -# Microsoft Azure Web App publish settings. Comment the next line if you want to -# checkin your Azure Web App publish settings, but sensitive information contained -# in these scripts will be unencrypted -PublishScripts/ - -# NuGet Packages -*.nupkg -# The packages folder can be ignored because of Package Restore -**/[Pp]ackages/* -# except build/, which is used as an MSBuild target. -!**/[Pp]ackages/build/ -# Uncomment if necessary however generally it will be regenerated when needed -#!**/[Pp]ackages/repositories.config -# NuGet v3's project.json files produces more ignorable files -*.nuget.props -*.nuget.targets - -# Microsoft Azure Build Output -csx/ -*.build.csdef - -# Microsoft Azure Emulator -ecf/ -rcf/ - -# Windows Store app package directories and files -AppPackages/ -BundleArtifacts/ -Package.StoreAssociation.xml -_pkginfo.txt -*.appx - -# Visual Studio cache files -# files ending in .cache can be ignored -*.[Cc]ache -# but keep track of directories ending in .cache -!*.[Cc]ache/ - -# Others -ClientBin/ -~$* -*~ -*.dbmdl -*.dbproj.schemaview -*.jfm -*.pfx -*.publishsettings -orleans.codegen.cs - -# Including strong name files can present a security risk -# (https://github.com/github/gitignore/pull/2483#issue-259490424) -#*.snk - -# Since there are multiple workflows, uncomment next line to ignore bower_components -# (https://github.com/github/gitignore/pull/1529#issuecomment-104372622) -#bower_components/ - -# RIA/Silverlight projects -Generated_Code/ - -# Backup & report files from converting an old project file -# to a newer Visual Studio version. Backup files are not needed, -# because we have git ;-) -_UpgradeReport_Files/ -Backup*/ -UpgradeLog*.XML -UpgradeLog*.htm -ServiceFabricBackup/ -*.rptproj.bak - -# SQL Server files -*.mdf -*.ldf -*.ndf - -# Business Intelligence projects -*.rdl.data -*.bim.layout -*.bim_*.settings -*.rptproj.rsuser - -# Microsoft Fakes -FakesAssemblies/ - -# GhostDoc plugin setting file -*.GhostDoc.xml - -# Node.js Tools for Visual Studio -.ntvs_analysis.dat -node_modules/ - -# Visual Studio 6 build log -*.plg - -# Visual Studio 6 workspace options file -*.opt - -# Visual Studio 6 auto-generated workspace file (contains which files were open etc.) -*.vbw - -# Visual Studio LightSwitch build output -**/*.HTMLClient/GeneratedArtifacts -**/*.DesktopClient/GeneratedArtifacts -**/*.DesktopClient/ModelManifest.xml -**/*.Server/GeneratedArtifacts -**/*.Server/ModelManifest.xml -_Pvt_Extensions - -# Paket dependency manager -.paket/paket.exe -paket-files/ - -# FAKE - F# Make -.fake/ - -# JetBrains Rider -.idea/ -*.sln.iml - -# CodeRush -.cr/ - -# Python Tools for Visual Studio (PTVS) -__pycache__/ -*.pyc - -# Cake - Uncomment if you are using it -# tools/** -# !tools/packages.config - -# Tabs Studio -*.tss - -# Telerik's JustMock configuration file -*.jmconfig +# Linker output +*.ilk +*.map +*.exp -# BizTalk build output -*.btp.cs -*.btm.cs -*.odx.cs -*.xsd.cs +# Precompiled Headers +*.gch +*.pch -# OpenCover UI analysis results -OpenCover/ +# Libraries +*.lib +*.a +*.la +*.lo + +# Shared objects (inc. Windows DLLs) +*.dll +*.so +*.so.* +*.dylib + +# Executables +*.out +*.app +*.i*86 +*.x86_64 +*.hex + +# Debug files +*.dSYM/ +*.su +*.idb +*.pdb -# Azure Stream Analytics local run output -ASALocalRun/ +# Kernel Module Compile Results +*.mod* +*.cmd +.tmp_versions/ +modules.order +Module.symvers +Mkfile.old +dkms.conf -# MSBuild Binary and Structured Log -*.binlog +# Cmake +build -# NVidia Nsight GPU debugger configuration file -*.nvuser +# VS folder +.vs/ -# MFractors (Xamarin productivity tool) working folder -.mfractor/ +# Test framework +test/catch2 \ No newline at end of file diff --git a/CMakeLists.txt b/CMakeLists.txt new file mode 100644 index 0000000..e8f8364 --- /dev/null +++ b/CMakeLists.txt @@ -0,0 +1,177 @@ +cmake_minimum_required(VERSION 2.8) + +project(azurestoragelite) + +option(BUILD_TESTS "Build test codes" OFF) +option(BUILD_SAMPLES "Build sample codes" OFF) + +set(AZURE_STORAGE_LITE_HEADER + include/storage_EXPORTS.h + + include/base64.h + include/common.h + include/constants.h + include/constants.dat + include/executor.h + include/hash.h + include/retry.h + include/utility.h + + include/tinyxml2.h + include/xml_parser_base.h + include/tinyxml2_parser.h + include/xml_writer.h + + include/storage_account.h + include/storage_credential.h + include/storage_outcome.h + include/storage_stream.h + include/storage_url.h + include/storage_errno.h + + include/storage_request_base.h + include/get_blob_request_base.h + include/put_blob_request_base.h + include/delete_blob_request_base.h + include/copy_blob_request_base.h + include/create_container_request_base.h + include/delete_container_request_base.h + include/list_containers_request_base.h + include/list_blobs_request_base.h + include/get_block_list_request_base.h + include/put_block_request_base.h + include/get_blob_property_request_base.h + include/get_container_property_request_base.h + include/put_block_list_request_base.h + include/get_container_property_request_base.h + include/append_block_request_base.h + include/put_page_request_base.h + include/get_page_ranges_request_base.h + + include/http_base.h + include/http/libcurl_http_client.h + + include/blob/blob_client.h + include/blob/download_blob_request.h + include/blob/create_block_blob_request.h + include/blob/delete_blob_request.h + include/blob/copy_blob_request.h + include/blob/create_container_request.h + include/blob/delete_container_request.h + include/blob/list_containers_request.h + include/blob/list_blobs_request.h + include/blob/get_blob_property_request.h + include/blob/get_container_property_request.h + include/blob/get_block_list_request.h + include/blob/put_block_request.h + include/blob/put_block_list_request.h + include/blob/append_block_request.h + include/blob/put_page_request.h + include/blob/get_page_ranges_request.h +) + +set(AZURE_STORAGE_LITE_SOURCE + src/base64.cpp + src/constants.cpp + src/hash.cpp + src/utility.cpp + + src/tinyxml2.cpp + src/tinyxml2_parser.cpp + + src/storage_account.cpp + src/storage_credential.cpp + src/storage_url.cpp + + src/get_blob_request_base.cpp + src/put_blob_request_base.cpp + src/delete_blob_request_base.cpp + src/copy_blob_request_base.cpp + src/create_container_request_base.cpp + src/delete_container_request_base.cpp + src/list_containers_request_base.cpp + src/list_blobs_request_base.cpp + src/get_blob_property_request_base.cpp + src/get_block_list_request_base.cpp + src/get_container_property_request_base.cpp + src/put_block_request_base.cpp + src/put_block_list_request_base.cpp + src/append_block_request_base.cpp + src/put_page_request_base.cpp + src/get_page_ranges_request_base.cpp + + src/http/libcurl_http_client.cpp + + src/blob/blob_client.cpp + src/blob/blob_client_wrapper.cpp +) + +find_package(CURL REQUIRED) + +option(USE_OPENSSL "Use OpenSSL instead of GnuTLS" OFF) + +find_package(Threads REQUIRED) +find_package(OpenSSL REQUIRED) +pkg_search_module(UUID REQUIRED uuid) + +if (NOT ${USE_OPENSSL}) + find_package(GnuTLS REQUIRED) + set(EXTRA_INCLUDE ${GNUTLS_INCLUDE_DIR}) + set(EXTRA_LIBRARIES ${GNUTLS_LIBRARIES}) +else() + add_definitions(-DUSE_OPENSSL) +endif() + +add_definitions(-std=c++11) +set(WARNING "-Wall -Wextra -Werror -pedantic -pedantic-errors") +set(CMAKE_CXX_FLAGS "${CMAKE_THREAD_LIBS_INIT} ${WARNING} ${CMAKE_CXX_FLAGS}") +include_directories(${CMAKE_SOURCE_DIR}/include ${CURL_INCLUDE_DIRS} ${OPENSSL_INCLUDE_DIR} ${UUID_INCLUDE_DIR} ${EXTRA_INCLUDE}) + +set(CMAKE_MACOSX_RPATH ON) +add_library(azure-storage-lite SHARED ${AZURE_STORAGE_LITE_HEADER} ${AZURE_STORAGE_LITE_SOURCE}) + +target_link_libraries(azure-storage-lite ${CURL_LIBRARIES} ${OPENSSL_CRYPTO_LIBRARY} ${OPENSSL_SSL_LIBRARY} ${UUID_LIBRARIES} ${EXTRA_LIBRARIES}) + +if(BUILD_TESTS) + add_subdirectory(test) +endif() + +if(BUILD_SAMPLES) + add_subdirectory(sample) +endif() + +if (WIN32 OR UNIX) + file(GLOB BLOB_HEADERS includes/blob/*.h) + install(FILES ${BLOB_HEADERS} DESTINATION include/blob) + + file(GLOB HTTP_HEADERS includes/http/*.h) + install(FILES ${HTTP_HEADERS} DESTINATION include/http) + + file(GLOB TODO_HEADERS includes/todo/*.h) + install(FILES ${HTTP_HEADERS} DESTINATION todo/http) + + file(GLOB GENERAL_HEADERS includes/*.h) + install(FILES ${GENERAL_HEADERS} DESTINATION include) + + file(GLOB GENERAL_DATA includes/*.dat) + install(FILES ${GENERAL_DATA} DESTINATION include) +endif() + +# Reconfigure final output directory +set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${PROJECT_BINARY_DIR}/Binaries) +set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${PROJECT_BINARY_DIR}/Binaries) +set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${PROJECT_BINARY_DIR}/Binaries) + +# Set version numbers centralized +set (AZURE_STORAGE_LITE_VERSION_MAJOR 0) +set (AZURE_STORAGE_LITE_VERSION_MINOR 1) +set (AZURE_STORAGE_LITE_VERSION_REVISION 0) + +# Set output directories. +if(NOT DEFINED CMAKE_INSTALL_BINDIR) + set(CMAKE_INSTALL_BINDIR bin) +endif() + +if(NOT DEFINED CMAKE_INSTALL_LIBDIR) + set(CMAKE_INSTALL_LIBDIR lib) +endif() diff --git a/CMakeSettings.json b/CMakeSettings.json new file mode 100644 index 0000000..ab28b59 --- /dev/null +++ b/CMakeSettings.json @@ -0,0 +1,58 @@ +{ + "configurations": [ + { + "name": "Linux-Release", + "generator": "Unix Makefiles", + "remoteMachineName": "${defaultRemoteMachineName}", + "remoteCopySourcesMethod": "sftp", + "configurationType": "Release", + "cmakeExecutable": "/usr/local/bin/cmake", + "remoteCMakeListsRoot": "/var/tmp/src/${workspaceHash}/${name}", + "buildRoot": "${env.LOCALAPPDATA}\\CMakeBuilds\\${workspaceHash}\\build\\${name}", + "remoteBuildRoot": "/var/tmp/build/${workspaceHash}/build/${name}", + "remoteCopySources": true, + "cmakeCommandArgs": "", + "buildCommandArgs": "-m", + "inheritEnvironments": [ "linux-x64" ], + "variables": [ + { + "name": "BUILD_TESTS", + "value": "true" + }, + { + "name": "BUILD_SAMPLES", + "value": "true" + } + ] + }, + { + "name": "Linux-Debug", + "generator": "Unix Makefiles", + "remoteMachineName": "${defaultRemoteMachineName}", + "remoteCopySourcesMethod": "sftp", + "configurationType": "Debug", + "remoteCMakeListsRoot": "/var/tmp/src/${workspaceHash}/${name}", + "cmakeExecutable": "/usr/local/bin/cmake", + "buildRoot": "${env.LOCALAPPDATA}\\CMakeBuilds\\${workspaceHash}\\build\\${name}", + "remoteBuildRoot": "/var/tmp/build/${workspaceHash}/build/${name}", + "remoteCopySources": true, + "remoteCopySourcesOutputVerbosity": "Normal", + "remoteCopySourcesConcurrentCopies": "10", + "cmakeCommandArgs": "", + "ctestCommandArgs": "", + "buildCommandArgs": "-m", + "inheritEnvironments": [ "linux-x64" ], + "variables": [ + { + "name": "BUILD_TESTS", + "value": "true" + }, + { + "name": "BUILD_SAMPLES", + "value": "true" + } + ] + } + ] +} + diff --git a/NOTICES.txt b/NOTICES.txt new file mode 100644 index 0000000..bea1fff --- /dev/null +++ b/NOTICES.txt @@ -0,0 +1,17 @@ +azure-storage-cpplite incorporates material from the project(s) listed below (“Third Party Code”). Microsoft is not the original author of the Third Party Code. Microsoft reserves all other rights not expressly granted, whether by implication, estoppel or otherwise. +Third-party notices are provided solely for your information and include the original copyright and license which Microsoft received with the third-party software. While Microsoft is not the original author of the third-party materials, Microsoft licenses these third-party materials to you under the terms set forth in the agreement governing the Microsoft Offering, except that components licensed under open source licenses requiring that such components remain under their original license, such as the GNU General Public License (GPL) or the GNU Lesser General Public License (LGPL), are being made available to you by Microsoft under their original licensing terms. Microsoft reserves all rights not expressly granted herein, whether by implication, estoppel or otherwise. + +TinyXML-2 + +=========== +TinyXML-2 is released under the zlib license: + +This software is provided 'as-is', without any express or implied warranty. In no event will the authors be held liable for any damages arising from the use of this software. + +Permission is granted to anyone to use this software for any purpose, including commercial applications, and to alter it and redistribute it freely, subject to the following restrictions: + +The origin of this software must not be misrepresented; you must not claim that you wrote the original software. If you use this software in a product, an acknowledgment in the product documentation would be appreciated but is not required. +Altered source versions must be plainly marked as such, and must not be misrepresented as being the original software. +This notice may not be removed or altered from any source distribution. +=========== +` diff --git a/README.md b/README.md index 72f1506..8b210cf 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,82 @@ +# Azure Storage C++ Client Library (Lite) -# Contributing +## About + +The Azure Storage Client Library (Lite) for C++ allows you to build applications against Microsoft Azure Storage's blob service. This is a minimum dependency version that provide basic object storage. For an overview of Azure Storage, see [Introduction to Microsoft Azure Storage](http://azure.microsoft.com/en-us/documentation/articles/storage-introduction/). +If you want to use other services of Azure Storage, or a more comprehensive functionality of Blob service, please see [Azure Storage C++ Client Library](https://github.com/azure/azure-storage-cpp). + +## Features +The full supported Azure Storage API can be found in the following list, please be aware that only part of the functionality of some APIs are supported: +- [List Containers](https://docs.microsoft.com/en-us/rest/api/storageservices/list-containers2). +- [Create Container](https://docs.microsoft.com/en-us/rest/api/storageservices/create-container). +- [Get Container Properties](https://docs.microsoft.com/en-us/rest/api/storageservices/get-container-properties). +- [Delete Container](https://docs.microsoft.com/en-us/rest/api/storageservices/delete-container). +- [List Blobs](https://docs.microsoft.com/en-us/rest/api/storageservices/list-blobs). +- [Put Blob](https://docs.microsoft.com/en-us/rest/api/storageservices/put-blob). +- [Get Blob](https://docs.microsoft.com/en-us/rest/api/storageservices/get-blob). +- [Get Blob Properties](https://docs.microsoft.com/en-us/rest/api/storageservices/get-blob-properties). +- [Delete Blob](https://docs.microsoft.com/en-us/rest/api/storageservices/delete-blob). +- [Copy Blob](https://docs.microsoft.com/en-us/rest/api/storageservices/copy-blob). +- [Put Block](https://docs.microsoft.com/en-us/rest/api/storageservices/put-block). +- [Put Block List](https://docs.microsoft.com/en-us/rest/api/storageservices/put-block-list). +- [Get Block List](https://docs.microsoft.com/en-us/rest/api/storageservices/get-block-list). +- [Put Page](https://docs.microsoft.com/en-us/rest/api/storageservices/put-page). +- [Get Page Ranges](https://docs.microsoft.com/en-us/rest/api/storageservices/get-page-ranges). +- [Append Block](https://docs.microsoft.com/en-us/rest/api/storageservices/append-block). + +## Installation + +### Clone the latest code from this repository: +``` +git clone https://github.com/azure/azure-storage-cpplite.git +``` +### Install the dependencies, e.g. on Ubuntu: +``` +sudo apt-get install libssl-dev libcurl4-openssl-dev cmake g++ +``` +Or, on Red Hat OS: +``` +sudo yum install openssl-devel.x86_64 libcurl-devel.x86_64 cmake.x86_64 gcc-c++.x86_64 +``` +Please be aware that RHEL6 comes with gcc version 4.4.7, which does not meet the requirement of this SDK. In order to use this SDK, [devtoolset](http://linux.web.cern.ch/linux/devtoolset/#install) needs to be installed properly. +### Build and install azure-storage-cpplite: +``` +cd azure-storage-cpplite +mkdir build.release +cd build.release +CXX=g++ cmake .. -DCMAKE_BUILD_TYPE=Release -DUSE_OPENSSL=true +sudo make install +``` +### Use GNUTLS instead of OpenSSL: +Alternatively, you can use GNUTLS instead of OpenSSL. Simply install GNUTLS and remove the argument `-DUSE_OPENSSL` during build. + +## Usage +Simply include the header files after installing the library, everything is good to go. For a more comprehensive sample, please see [sample](https://github.com/azure/azure-storage-cpplite/blob/master/sample/sample.cpp). +To build the sample, add `-DBUILD_SAMPLES=true` when building the repository. +``` +#include "storage_credential.h" +#include "storage_account.h" +#include "blob/blob_client.h" + +// Your settings +std::string account_name = "YOUR_ACCOUNT_NAME"; +std::string account_key = "YOUR_ACCOUNT_KEY"; +bool use_https = true; +std::string blob_endpoint = "CUSTOMIZED_BLOB_ENDPOINT"; +int connection_count = 2; + +// Setup the client +azure::storage_lite::shared_key_credential credential(account_name, account_key); +azure::storage_lite::storage_account(account_name, credential, use_https, blob_endpoint); +azure::storage_lite::blob_client client(storage_account, connection_count); + +// Start using +auto outcome = client.create_container("YOUR_CONTAINER_NAME").get(); +``` +## License +This project is licensed under MIT. + +## Contributing This project welcomes contributions and suggestions. Most contributions require you to agree to a Contributor License Agreement (CLA) declaring that you have the right to, and actually do, grant us @@ -12,3 +89,23 @@ provided by the bot. You will only need to do this once across all repos using o This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/). For more information see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) or contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments. + +When contributing to this client library, there are following ground rules: +1. All source code change should be clearly addressing issues or adding new features, and should be covered with test. +2. Coding style should match with the existing code. +3. Any contribution should not degrade performance or complex functionality. +4. Introducing new dependency should be done with much great caution. Any new dependency should introduce significant performance improvement or unblock critical user scenario. + +### Build Test +Download [Catch2 single header version](https://raw.githubusercontent.com/catchorg/Catch2/master/single_include/catch2/catch.hpp) and put it in the folder `.\test\catch2\`. +Add `-DBUILD_TESTS=true` when building the repository. +Please modify the [connection string here](https://github.com/katmsft/azure-storage-cpplite/blob/master/test/test_base.h#L18) to successfully run the tests. All the test uses standard Azure Storage account. +#### *Please note that in order to run test, a minimum version of g++ 5.1 is required.* + +### Dependencies +- Project dependencies: + - GNUTLS(or Openssl v7.35.0) + - libcurl v1.0.1 + - CMake v2.8.12.2 + - g++ v4.8.2 +- Dev dependency: catch2. diff --git a/include/append_block_request_base.h b/include/append_block_request_base.h new file mode 100644 index 0000000..e8d2c36 --- /dev/null +++ b/include/append_block_request_base.h @@ -0,0 +1,29 @@ +#pragma once + +#include +#include + +#include "storage_EXPORTS.h" + +#include "http_base.h" +#include "storage_account.h" +#include "storage_request_base.h" + +namespace azure { namespace storage_lite { + + class append_block_request_base : public blob_request_base + { + public: + virtual std::string container() const = 0; + virtual std::string blob() const = 0; + + virtual unsigned int content_length() const = 0; + virtual std::string content_md5() const { return std::string(); } + + virtual unsigned long long ms_blob_condition_maxsize() const { return 0; } + virtual unsigned long long ms_blob_condition_appendpos() const { return 0; } + + AZURE_STORAGE_API void build_request(const storage_account &a, http_base &h) const override; + }; + +}} diff --git a/include/base64.h b/include/base64.h new file mode 100644 index 0000000..6f22f45 --- /dev/null +++ b/include/base64.h @@ -0,0 +1,13 @@ +#pragma once + +#include +#include + +#include "storage_EXPORTS.h" + +namespace azure { namespace storage_lite { + + AZURE_STORAGE_API std::string to_base64(const std::vector &input); + AZURE_STORAGE_API std::vector from_base64(const std::string &input); + +}} diff --git a/include/blob/append_block_request.h b/include/blob/append_block_request.h new file mode 100644 index 0000000..2b66a6b --- /dev/null +++ b/include/blob/append_block_request.h @@ -0,0 +1,42 @@ +#pragma once + +#include "append_block_request_base.h" + +namespace azure { namespace storage_lite { + + class append_block_request : public append_block_request_base + { + public: + append_block_request(const std::string &container, const std::string &blob) + : m_container(container), + m_blob(blob), + m_content_length(0) {} + + std::string container() const override + { + return m_container; + } + + std::string blob() const override + { + return m_blob; + } + + unsigned int content_length() const override + { + return m_content_length; + } + + append_block_request &set_content_length(unsigned int content_length) + { + m_content_length = content_length; + return *this; + } + + private: + std::string m_container; + std::string m_blob; + + unsigned int m_content_length; + }; +}} // azure::storage_lite diff --git a/include/blob/blob_client.h b/include/blob/blob_client.h new file mode 100644 index 0000000..fe6ca1b --- /dev/null +++ b/include/blob/blob_client.h @@ -0,0 +1,463 @@ +#pragma once + +#include +#include +#include +#ifdef __linux__ +#include +#endif + +#include "storage_EXPORTS.h" + +#include "storage_account.h" +#include "http/libcurl_http_client.h" +#include "tinyxml2_parser.h" +#include "executor.h" +#include "put_block_list_request_base.h" +#include "get_blob_property_request_base.h" +#include "get_blob_request_base.h" +#include "get_container_property_request_base.h" +#include "list_blobs_request_base.h" + +namespace azure { namespace storage_lite { + + /// + /// Provides a client-side logical representation of blob storage service on Windows Azure. This client is used to configure and execute requests against the service. + /// + /// The service client encapsulates the base URI for the service. If the service client will be used for authenticated access, it also encapsulates the credentials for accessing the storage account. + class blob_client + { + public: + /// + /// Initializes a new instance of the class. + /// + /// An existing object. + /// An int value indicates the maximum concurrency expected during execute requests against the service. + blob_client(std::shared_ptr account, int max_concurrency) + : m_account(account) + { + m_context = std::make_shared(std::make_shared(), std::make_shared()); + m_client = std::make_shared(max_concurrency); + } + + /// + /// Gets the curl client used to execute requests. + /// + /// The object + std::shared_ptr client() const + { + return m_client; + } + + /// + /// Gets the storage account used to store the base uri and credentails. + /// + std::shared_ptr account() const + { + return m_account; + } + + /// + /// Gets the max parallelism used. + /// + unsigned int concurrency() const + { + return m_client->size(); + } + + /// + /// Synchronously download the contents of a blob to a stream. + /// + /// The container name. + /// The blob name. + /// The offset at which to begin downloading the blob, in bytes. + /// The size of the data to download from the blob, in bytes. + /// The target stream. + /// A object that represents the current operation. + AZURE_STORAGE_API storage_outcome get_chunk_to_stream_sync(const std::string &container, const std::string &blob, unsigned long long offset, unsigned long long size, std::ostream &os); + + /// + /// Intitiates an asynchronous operation to download the contents of a blob to a stream. + /// + /// The container name. + /// The blob name. + /// The offset at which to begin downloading the blob, in bytes. + /// The size of the data to download from the blob, in bytes. + /// The target stream. + /// A object that represents the current operation. + AZURE_STORAGE_API std::future> download_blob_to_stream(const std::string &container, const std::string &blob, unsigned long long offset, unsigned long long size, std::ostream &os); + + /// + /// Intitiates an asynchronous operation to upload the contents of a blob from a stream. + /// + /// The container name. + /// The blob name. + /// The source stream. + /// A that respresents metadatas. + /// A object that represents the current operation. + AZURE_STORAGE_API std::future> upload_block_blob_from_stream(const std::string &container, const std::string &blob, std::istream &is, const std::vector> &metadata); + + /// + /// Intitiates an asynchronous operation to delete a blob. + /// + /// The container name. + /// The blob name. + /// A bool value, delete snapshots if it is true. + /// A object that represents the current operation. + AZURE_STORAGE_API std::future> delete_blob(const std::string &container, const std::string &blob, bool delete_snapshots = false); + + /// + /// Intitiates an asynchronous operation to create a container. + /// + /// The container name. + /// A object that represents the current operation. + AZURE_STORAGE_API std::future> create_container(const std::string &container); + + /// + /// Intitiates an asynchronous operation to delete a container. + /// + /// The container name. + /// A object that represents the current operation. + AZURE_STORAGE_API std::future> delete_container(const std::string &container); + + /// + /// Intitiates a synchronous operation to get the container property. + /// + /// The container name. + /// A object that represents the current operation's result. + AZURE_STORAGE_API storage_outcome get_container_property(const std::string &container); + + /// + /// Intitiates an asynchronous operation to list containers. + /// + /// The container name prefix. + /// A bool value, return metadatas if it is true. + /// A object that represents the current operation. + AZURE_STORAGE_API std::future> list_containers_segmented(const std::string &prefix, const std::string& continuation_token, const int max_result = 5, bool include_metadata = false); + + /// + /// Intitiates an asynchronous operation to list blobs in segments. + /// + /// The container name. + /// The delimiter used to designate the virtual directories. + /// A continuation token returned by a previous listing operation. + /// The blob name prefix. + /// A object that represents the current operation. + AZURE_STORAGE_API std::future> list_blobs_segmented(const std::string &container, const std::string &delimiter, const std::string &continuation_token, const std::string &prefix, int max_results = 10000); + + /// + /// Intitiates an asynchronous operation to get the property of a blob. + /// + /// The container name. + /// The blob name. + /// A object that represents the current operation. + AZURE_STORAGE_API storage_outcome get_blob_property(const std::string &container, const std::string &blob); + + /// + /// Intitiates an asynchronous operation to download the block list of a blob. + /// + /// The container name. + /// The blob name. + /// A object that represents the current operation. + AZURE_STORAGE_API std::future> get_block_list(const std::string &container, const std::string &blob); + + /// + /// Intitiates an asynchronous operation to upload a block of a blob from a stream. + /// + /// The container name. + /// The blob name. + /// A Base64-encoded block ID that identifies the block. + /// The source stream. + /// A object that represents the current operation. + AZURE_STORAGE_API std::future> upload_block_from_stream(const std::string &container, const std::string &blob, const std::string &blockid, std::istream &is); + + /// + /// Intitiates an asynchronous operation to create a block blob with existing blocks. + /// + /// The container name. + /// The blob name. + /// A that contains all blocks in order. + /// A that respresents metadatas. + /// A object that represents the current operation. + AZURE_STORAGE_API std::future> put_block_list(const std::string &container, const std::string &blob, const std::vector &block_list, const std::vector> &metadata); + + /// + /// Intitiates an asynchronous operation to create an append blob. + /// + /// The container name. + /// The blob name. + /// A object that represents the current operation. + AZURE_STORAGE_API std::future> create_append_blob(const std::string &container, const std::string &blob); + + /// + /// Intitiates an asynchronous operation to append the content to an append blob from a stream. + /// + /// The container name. + /// The blob name. + /// The source stream. + /// A object that represents the current operation. + AZURE_STORAGE_API std::future> append_block_from_stream(const std::string &container, const std::string &blob, std::istream &is); + + /// + /// Intitiates an asynchronous operation to create an page blob. + /// + /// The container name. + /// The blob name. + /// The size of the page blob, in bytes. + /// A object that represents the current operation. + AZURE_STORAGE_API std::future> create_page_blob(const std::string &container, const std::string &blob, unsigned long long size); + + /// + /// Intitiates an asynchronous operation to upload a blob range content from a stream. + /// + /// The container name. + /// The blob name. + /// The offset at which to begin upload to the blob, in bytes. + /// The size of the data, in bytes. + /// The target stream. + /// A object that represents the current operation. + AZURE_STORAGE_API std::future> put_page_from_stream(const std::string &container, const std::string &blob, unsigned long long offset, unsigned long long size, std::istream &is); + + /// + /// Intitiates an asynchronous operation to clear pages of a page blob range. + /// + /// The container name. + /// The blob name. + /// The offset at which to begin clearing, in bytes. + /// The size of the data to be cleared from the blob, in bytes. + /// A object that represents the current operation. + AZURE_STORAGE_API std::future> clear_page(const std::string &container, const std::string &blob, unsigned long long offset, unsigned long long size); + + /// + /// Intitiates an asynchronous operation to get the page ranges fro a page blob. + /// + /// The container name. + /// The blob name. + /// The offset at which to get, in bytes. + /// The size of the data to be get from the blob, in bytes. + /// A object that represents the current operation. + AZURE_STORAGE_API std::future> get_page_ranges(const std::string &container, const std::string &blob, unsigned long long offset, unsigned long long size); + + /// + /// Intitiates an asynchronous operation to copy a blob to another. + /// + /// The source container name. + /// The source blob name. + /// The destination container name. + /// The destination blob name. + /// A object that represents the current operation. + AZURE_STORAGE_API std::future> start_copy(const std::string &sourceContainer, const std::string &sourceBlob, const std::string &destContainer, const std::string &destBlob); + + private: + std::shared_ptr m_client; + std::shared_ptr m_account; + std::shared_ptr m_context; + }; + + /// + /// Provides a wrapper for client-side logical representation of blob storage service on Windows Azure. This wrappered client is used to configure and execute requests against the service. + /// + /// This wrappered client could limit a concurrency per client objects. And it will not throw exceptions, instead, it will set errno to return error codes. + class blob_client_wrapper + { + public: + /// + /// Constructs a blob client wrapper from a blob client instance. + /// + /// A object stored in shared_ptr. + explicit blob_client_wrapper(std::shared_ptr blobClient) + : m_blobClient(blobClient), + m_valid(true) + { + if (blobClient != NULL) + { + m_concurrency = blobClient->concurrency(); + } + } + + /// + /// Constructs an empty blob client wrapper. + /// + /// A bool value indicates this client wrapper is valid or not. + explicit blob_client_wrapper(bool valid) + : m_valid(valid) + { + } + + /// + /// Constructs a blob client wrapper from another blob client wrapper instance. + /// + /// A object. + blob_client_wrapper(blob_client_wrapper &&other) + { + m_blobClient = other.m_blobClient; + m_concurrency = other.m_concurrency; + m_valid = other.m_valid; + } + + blob_client_wrapper& operator=(blob_client_wrapper&& other) + { + m_blobClient = other.m_blobClient; + m_concurrency = other.m_concurrency; + m_valid = other.m_valid; + return *this; + } + + bool is_valid() const + { + return m_valid && (m_blobClient != NULL); + } + + /// + /// Constructs a blob client wrapper from storage account credential. + /// + /// The storage account name. + /// The storage account key. + /// A sas token for the container. + /// The maximum number requests could be executed in the same time. + /// Return a object. + static blob_client_wrapper blob_client_wrapper_init(const std::string &account_name, const std::string &account_key, const std::string &sas_token, const unsigned int concurrency); + + /// + /// Constructs a blob client wrapper from storage account credential. + /// + /// The storage account name. + /// The storage account key. + /// A sas token for the container. + /// The maximum number requests could be executed in the same time. + /// True if https should be used (instead of HTTP). Note that this may cause a sizable perf loss, due to issues in libcurl. + /// Blob endpoint URI to allow non-public clouds as well as custom domains. + /// Return a object. + static blob_client_wrapper blob_client_wrapper_init(const std::string &account_name, const std::string &account_key, const std::string &sas_token, const unsigned int concurrency, bool use_https, + const std::string &blob_endpoint); + /* C++ wrappers without exception but error codes instead */ + + /* container level*/ + + /// + /// Creates a container. + /// + /// The container name. + void create_container(const std::string &container); + + /// + /// Deletes a container. + /// + /// The container name. + void delete_container(const std::string &container); + + /// + /// Examines the existance of a container. + /// + /// The container name. + /// Return true if the container does exist, otherwise, return false. + bool container_exists(const std::string &container); + + /// + /// List containers. + /// + /// The container name prefix. + /// A bool value, return metadatas if it is true. + std::vector list_containers_segmented(const std::string &prefix, const std::string& continuation_token, const int max_result = 5, bool include_metadata = false); + + /* blob level */ + + /// + /// List blobs in segments. + /// + /// The container name. + /// The delimiter used to designate the virtual directories. + /// A continuation token returned by a previous listing operation. + /// The blob name prefix. + list_blobs_segmented_response list_blobs_segmented(const std::string &container, const std::string &delimiter, const std::string &continuation_token, const std::string &prefix, int maxresults = 10000); + + /// + /// Uploads the contents of a blob from a local file, file size need to be equal or smaller than 64MB. + /// + /// The source file path. + /// The container name. + /// The blob name. + /// A that respresents metadatas. + void put_blob(const std::string &sourcePath, const std::string &container, const std::string blob, const std::vector> &metadata = std::vector>()); + + /// + /// Uploads the contents of a blob from a stream. + /// + /// The container name. + /// The blob name. + /// The source stream. + /// A that respresents metadatas. + void upload_block_blob_from_stream(const std::string &container, const std::string blob, std::istream &is, const std::vector> &metadata = std::vector>()); + + /// + /// Uploads the contents of a blob from a local file. + /// + /// The source file path. + /// The container name. + /// The blob name. + /// A that respresents metadatas. + /// A size_t value indicates the maximum parallelism can be used in this request. + void upload_file_to_blob(const std::string &sourcePath, const std::string &container, const std::string blob, const std::vector> &metadata = std::vector>(), size_t parallel = 8); + + /// + /// Downloads the contents of a blob to a stream. + /// + /// The container name. + /// The blob name. + /// The offset at which to begin downloading the blob, in bytes. + /// The size of the data to download from the blob, in bytes. + /// The target stream. + void download_blob_to_stream(const std::string &container, const std::string &blob, unsigned long long offset, unsigned long long size, std::ostream &os); + + /// + /// Downloads the contents of a blob to a local file. + /// + /// The container name. + /// The blob name. + /// The offset at which to begin downloading the blob, in bytes. + /// The size of the data to download from the blob, in bytes. + /// The target file path. + /// A size_t value indicates the maximum parallelism can be used in this request. + /// A object that represents the properties (etag, last modified time and size) from the first chunk retrieved. + void download_blob_to_file(const std::string &container, const std::string &blob, const std::string &destPath, time_t &returned_last_modified, size_t parallel = 9); + + /// + /// Gets the property of a blob. + /// + /// The container name. + /// The blob name. + blob_property get_blob_property(const std::string &container, const std::string &blob); + + /// + /// Examines the existance of a blob. + /// + /// The container name. + /// The blob name. + /// Return true if the blob does exist, otherwise, return false. + bool blob_exists(const std::string &container, const std::string &blob); + + /// + /// Deletes a blob. + /// + /// The container name. + /// The blob name. + void delete_blob(const std::string &container, const std::string &blob); + + /// + /// Copy a blob to another. + /// + /// The source container name. + /// The source blob name. + /// The destination container name. + /// The destination blob name. + void start_copy(const std::string &sourceContainer, const std::string &sourceBlob, const std::string &destContainer, const std::string &destBlob); + private: + blob_client_wrapper() {} + + std::shared_ptr m_blobClient; + std::mutex s_mutex; + unsigned int m_concurrency; + bool m_valid; + }; + +} } // azure::storage_lite diff --git a/include/blob/copy_blob_request.h b/include/blob/copy_blob_request.h new file mode 100644 index 0000000..a616349 --- /dev/null +++ b/include/blob/copy_blob_request.h @@ -0,0 +1,43 @@ +#pragma once + +#include "copy_blob_request_base.h" + +namespace azure { namespace storage_lite { + + class copy_blob_request : public copy_blob_request_base + { + public: + copy_blob_request(const std::string &container, const std::string &blob, const std::string &destContainer, const std::string &destBlob) + : m_container(container), + m_blob(blob), + m_destContainer(destContainer), + m_destBlob(destBlob) {} + + std::string container() const override + { + return m_container; + } + + std::string blob() const override + { + return m_blob; + } + + std::string destContainer() const override + { + return m_destContainer; + } + + std::string destBlob() const override + { + return m_destBlob; + } + + private: + std::string m_container; + std::string m_blob; + std::string m_destContainer; + std::string m_destBlob; + }; + +}} // azure::storage_lite diff --git a/include/blob/create_block_blob_request.h b/include/blob/create_block_blob_request.h new file mode 100644 index 0000000..28af04d --- /dev/null +++ b/include/blob/create_block_blob_request.h @@ -0,0 +1,103 @@ +#pragma once + +#include "put_blob_request_base.h" + +namespace azure { namespace storage_lite { + + class create_block_blob_request : public put_blob_request_base + { + public: + create_block_blob_request(const std::string &container, const std::string &blob) + : m_container(container), + m_blob(blob), + m_content_length(0) {} + + std::string container() const override + { + return m_container; + } + + std::string blob() const override + { + return m_blob; + } + + blob_type ms_blob_type() const override + { + return blob_type::block_blob; + } + + unsigned int content_length() const override + { + return m_content_length; + } + + create_block_blob_request &set_content_length(unsigned int content_length) + { + m_content_length = content_length; + return *this; + } + + std::vector> metadata() const override + { + return m_metadata; + } + + create_block_blob_request &set_metadata(const std::vector> &metadata) + { + m_metadata = metadata; + return *this; + } + + + private: + std::string m_container; + std::string m_blob; + + unsigned int m_content_length; + std::vector> m_metadata; + }; + + class create_append_blob_request : public create_block_blob_request + { + public: + create_append_blob_request(const std::string &container, const std::string &blob) + : create_block_blob_request(container, blob) {} + + blob_type ms_blob_type() const override + { + return blob_type::append_blob; + } + + unsigned int content_length() const override + { + return 0; + } + }; + + class create_page_blob_request : public create_block_blob_request { + public: + create_page_blob_request(const std::string &container, const std::string &blob, unsigned long long size) + : create_block_blob_request(container, blob), + m_ms_blob_content_length(size) {} + + blob_type ms_blob_type() const override + { + return blob_type::page_blob; + } + + unsigned int content_length() const override + { + return 0; + } + + unsigned long long ms_blob_content_length() const override + { + return m_ms_blob_content_length; + } + + private: + unsigned long long m_ms_blob_content_length; + }; + +}} // azure::storage_lite diff --git a/include/blob/create_container_request.h b/include/blob/create_container_request.h new file mode 100644 index 0000000..44fd4b8 --- /dev/null +++ b/include/blob/create_container_request.h @@ -0,0 +1,29 @@ +#pragma once + +#include "create_container_request_base.h" + +namespace azure { namespace storage_lite { + + class create_container_request : public create_container_request_base + { + public: + create_container_request(const std::string &container, blob_public_access public_access = blob_public_access::unspecified) + : m_container(container), + m_blob_public_access(public_access) {} + + std::string container() const override + { + return m_container; + } + + blob_public_access ms_blob_public_access() const override + { + return m_blob_public_access; + } + + private: + std::string m_container; + blob_public_access m_blob_public_access; + }; + +}} // azure::storage_lite diff --git a/include/blob/delete_blob_request.h b/include/blob/delete_blob_request.h new file mode 100644 index 0000000..af1fba9 --- /dev/null +++ b/include/blob/delete_blob_request.h @@ -0,0 +1,41 @@ +#pragma once + +#include "delete_blob_request_base.h" + +namespace azure { namespace storage_lite { + + class delete_blob_request : public delete_blob_request_base + { + public: + delete_blob_request(const std::string &container, const std::string &blob, bool delete_snapshots_only = false) + : m_container(container), + m_blob(blob), + m_delete_snapshots_only(delete_snapshots_only) {} + + std::string container() const override + { + return m_container; + } + + std::string blob() const override + { + return m_blob; + } + + delete_snapshots ms_delete_snapshots() const override + { + if (m_delete_snapshots_only) { + return delete_snapshots::only; + } + else { + return delete_snapshots::include; + } + } + + private: + std::string m_container; + std::string m_blob; + bool m_delete_snapshots_only; + }; + +}} // azure::storage_lite diff --git a/include/blob/delete_container_request.h b/include/blob/delete_container_request.h new file mode 100644 index 0000000..c84ec58 --- /dev/null +++ b/include/blob/delete_container_request.h @@ -0,0 +1,21 @@ +#pragma once + +#include "delete_container_request_base.h" + +namespace azure { namespace storage_lite { + + class delete_container_request : public delete_container_request_base + { + public: + delete_container_request(const std::string &container) + : m_container(container) {} + + std::string container() const override { + return m_container; + } + + private: + std::string m_container; + }; + +}} // azure::storage_lite diff --git a/include/blob/download_blob_request.h b/include/blob/download_blob_request.h new file mode 100644 index 0000000..7db079b --- /dev/null +++ b/include/blob/download_blob_request.h @@ -0,0 +1,54 @@ +#pragma once + +#include "get_blob_request_base.h" + +namespace azure { namespace storage_lite { + + class download_blob_request : public get_blob_request_base + { + public: + download_blob_request(const std::string &container, const std::string &blob) + : m_container(container), + m_blob(blob), + m_start_byte(0), + m_end_byte(0) {} + + std::string container() const override + { + return m_container; + } + + std::string blob() const override + { + return m_blob; + } + + unsigned long long start_byte() const override + { + return m_start_byte; + } + + unsigned long long end_byte() const override + { + return m_end_byte; + } + + download_blob_request &set_start_byte(unsigned long long start_byte) + { + m_start_byte = start_byte; + return *this; + } + + download_blob_request &set_end_byte(unsigned long long end_byte) + { + m_end_byte = end_byte; + return *this; + } + + private: + std::string m_container; + std::string m_blob; + unsigned long long m_start_byte; + unsigned long long m_end_byte; + }; +}} // azure::storage_lite diff --git a/include/blob/get_blob_property_request.h b/include/blob/get_blob_property_request.h new file mode 100644 index 0000000..9409cb4 --- /dev/null +++ b/include/blob/get_blob_property_request.h @@ -0,0 +1,29 @@ +#pragma once + +#include "get_blob_property_request_base.h" + +namespace azure { namespace storage_lite { + + class get_blob_property_request : public get_blob_property_request_base + { + public: + get_blob_property_request(const std::string &container, const std::string &blob) + : m_container(container), + m_blob(blob) {} + + std::string container() const override + { + return m_container; + } + + std::string blob() const override + { + return m_blob; + } + + private: + std::string m_container; + std::string m_blob; + }; + +}} // azure::storage_lite diff --git a/include/blob/get_block_list_request.h b/include/blob/get_block_list_request.h new file mode 100644 index 0000000..d966815 --- /dev/null +++ b/include/blob/get_block_list_request.h @@ -0,0 +1,29 @@ +#pragma once + +#include "get_block_list_request_base.h" + +namespace azure { namespace storage_lite { + + class get_block_list_request : public get_block_list_request_base + { + public: + get_block_list_request(const std::string &container, const std::string &blob) + : m_container(container), + m_blob(blob) {} + + std::string container() const override + { + return m_container; + } + + std::string blob() const override + { + return m_blob; + } + + private: + std::string m_container; + std::string m_blob; + }; + +}} // azure::storage_lite diff --git a/include/blob/get_container_property_request.h b/include/blob/get_container_property_request.h new file mode 100644 index 0000000..861c542 --- /dev/null +++ b/include/blob/get_container_property_request.h @@ -0,0 +1,23 @@ +#pragma once + +#include "get_container_property_request_base.h" + +namespace azure { namespace storage_lite { + + class get_container_property_request : public get_container_property_request_base + { + public: + get_container_property_request(const std::string &container) + : m_container(container) + {} + + std::string container() const override + { + return m_container; + } + + private: + std::string m_container; + }; + +}} // azure::storage_lite diff --git a/include/blob/get_page_ranges_request.h b/include/blob/get_page_ranges_request.h new file mode 100644 index 0000000..ddbb1c3 --- /dev/null +++ b/include/blob/get_page_ranges_request.h @@ -0,0 +1,55 @@ +#pragma once + +#include "get_page_ranges_request_base.h" + +namespace azure { namespace storage_lite { + + class get_page_ranges_request : public get_page_ranges_request_base + { + public: + get_page_ranges_request(const std::string &container, const std::string &blob) + : m_container(container), + m_blob(blob), + m_start_byte(0), + m_end_byte(0) {} + + std::string container() const override + { + return m_container; + } + + std::string blob() const override + { + return m_blob; + } + + unsigned long long start_byte() const override + { + return m_start_byte; + } + + unsigned long long end_byte() const override + { + return m_end_byte; + } + + get_page_ranges_request &set_start_byte(unsigned long long start_byte) + { + m_start_byte = start_byte; + return *this; + } + + get_page_ranges_request &set_end_byte(unsigned long long end_byte) + { + m_end_byte = end_byte; + return *this; + } + + private: + std::string m_container; + std::string m_blob; + unsigned long long m_start_byte; + unsigned long long m_end_byte; + }; + +}} // azure::storage_lite diff --git a/include/blob/list_blobs_request.h b/include/blob/list_blobs_request.h new file mode 100644 index 0000000..79fefbc --- /dev/null +++ b/include/blob/list_blobs_request.h @@ -0,0 +1,120 @@ +#pragma once + +#include "list_blobs_request_base.h" + +namespace azure { namespace storage_lite { + + class list_blobs_request : public list_blobs_request_base + { + public: + list_blobs_request(const std::string &container, const std::string &prefix) + : m_container(container), + m_prefix(prefix) {} + + std::string container() const override + { + return m_container; + } + + std::string prefix() const override + { + return m_prefix; + } + + std::string marker() const override + { + return m_marker; + } + + int maxresults() const override + { + return m_maxresults; + } + + list_blobs_request &set_marker(const std::string &marker) + { + m_marker = marker; + return *this; + } + + list_blobs_request &set_maxresults(int maxresults) + { + m_maxresults = maxresults; + return *this; + } + + private: + std::string m_container; + std::string m_prefix; + std::string m_marker; + int m_maxresults; + }; + + class list_blobs_segmented_request : public list_blobs_segmented_request_base + { + public: + list_blobs_segmented_request(const std::string &container, const std::string &delimiter, const std::string &continuation_token, const std::string &prefix) + : m_container(container), + m_prefix(prefix), + m_marker(continuation_token), + m_delimiter(delimiter), + m_maxresults(0) {} + + std::string container() const override + { + return m_container; + } + + std::string prefix() const override + { + return m_prefix; + } + + std::string marker() const override + { + return m_marker; + } + + std::string delimiter() const override + { + return m_delimiter; + } + + int maxresults() const override + { + return m_maxresults; + } + + list_blobs_request_base::include includes() const override + { + return m_includes; + } + + + list_blobs_segmented_request &set_marker(const std::string &marker) + { + m_marker = marker; + return *this; + } + + list_blobs_segmented_request &set_maxresults(int maxresults) + { + m_maxresults = maxresults; + return *this; + } + + list_blobs_segmented_request &set_includes(list_blobs_request_base::include includes) + { + m_includes = includes; + return *this; + } + + private: + std::string m_container; + std::string m_prefix; + std::string m_marker; + std::string m_delimiter; + int m_maxresults; + list_blobs_request_base::include m_includes; + }; +}} // azure::storage_lite diff --git a/include/blob/list_containers_request.h b/include/blob/list_containers_request.h new file mode 100644 index 0000000..873de90 --- /dev/null +++ b/include/blob/list_containers_request.h @@ -0,0 +1,53 @@ +#pragma once + +#include "list_containers_request_base.h" + +namespace azure { namespace storage_lite { + + class list_containers_request : public list_containers_request_base + { + public: + list_containers_request(const std::string &prefix, bool include_metadata = false) + : m_prefix(prefix), + m_include_metadata(include_metadata) {} + + std::string prefix() const override + { + return m_prefix; + } + + std::string marker() const override + { + return m_marker; + } + + int maxresults() const override + { + return m_maxresults; + } + + bool include_metadata() const override + { + return m_include_metadata; + } + + list_containers_request &set_marker(const std::string &marker) + { + m_marker = marker; + return *this; + } + + list_containers_request &set_maxresults(int maxresults) + { + m_maxresults = maxresults; + return *this; + } + + private: + std::string m_prefix; + std::string m_marker; + int m_maxresults; + bool m_include_metadata; + }; + +}} // azure::storage_lite diff --git a/include/blob/put_block_list_request.h b/include/blob/put_block_list_request.h new file mode 100644 index 0000000..8ac5a0b --- /dev/null +++ b/include/blob/put_block_list_request.h @@ -0,0 +1,53 @@ +#pragma once + +#include "put_block_list_request_base.h" + +namespace azure { namespace storage_lite { + + class put_block_list_request : public put_block_list_request_base + { + public: + put_block_list_request(const std::string &container, const std::string &blob) + : m_container(container), + m_blob(blob) {} + + std::string container() const override + { + return m_container; + } + + std::string blob() const override + { + return m_blob; + } + + std::vector block_list() const override + { + return m_block_list; + } + + put_block_list_request &set_block_list(const std::vector &block_list) + { + m_block_list = block_list; + return *this; + } + + std::vector> metadata() const override + { + return m_metadata; + } + + put_block_list_request &set_metadata(const std::vector> &metadata) + { + m_metadata = metadata; + return *this; + } + + private: + std::string m_container; + std::string m_blob; + std::vector m_block_list; + std::vector> m_metadata; + }; + +}} // azure::storage_lite diff --git a/include/blob/put_block_request.h b/include/blob/put_block_request.h new file mode 100644 index 0000000..ceebca4 --- /dev/null +++ b/include/blob/put_block_request.h @@ -0,0 +1,50 @@ +#pragma once + +#include "put_block_request_base.h" + +namespace azure { namespace storage_lite { + + class put_block_request : public put_block_request_base + { + public: + put_block_request(const std::string &container, const std::string &blob, const std::string &blockid) + : m_container(container), + m_blob(blob), + m_blockid(blockid), + m_content_length(0) {} + + std::string container() const override + { + return m_container; + } + + std::string blob() const override + { + return m_blob; + } + + std::string blockid() const override + { + return m_blockid; + } + + unsigned int content_length() const override + { + return m_content_length; + } + + put_block_request &set_content_length(unsigned int content_length) + { + m_content_length = content_length; + return *this; + } + + private: + std::string m_container; + std::string m_blob; + std::string m_blockid; + + unsigned int m_content_length; + }; + +}} // azure::storage_lite diff --git a/include/blob/put_page_request.h b/include/blob/put_page_request.h new file mode 100644 index 0000000..46b697f --- /dev/null +++ b/include/blob/put_page_request.h @@ -0,0 +1,77 @@ +#pragma once + +#include "put_page_request_base.h" + +namespace azure { namespace storage_lite { + + class put_page_request : public put_page_request_base + { + public: + put_page_request(const std::string &container, const std::string &blob, bool clear = false) + : m_container(container), + m_blob(blob), + m_clear(clear), + m_start_byte(0), + m_end_byte(0), + m_content_length(0) {} + + std::string container() const override + { + return m_container; + } + + std::string blob() const override + { + return m_blob; + } + + unsigned long long start_byte() const override + { + return m_start_byte; + } + + unsigned long long end_byte() const override + { + return m_end_byte; + } + + put_page_request &set_start_byte(unsigned long long start_byte) + { + m_start_byte = start_byte; + return *this; + } + + put_page_request &set_end_byte(unsigned long long end_byte) + { + m_end_byte = end_byte; + return *this; + } + + page_write ms_page_write() const override + { + if (m_clear) { + return page_write::clear; + } + return page_write::update; + } + + unsigned int content_length() const override + { + return m_content_length; + } + + put_page_request &set_content_length(unsigned int content_length) + { + m_content_length = content_length; + return *this; + } + + private: + std::string m_container; + std::string m_blob; + bool m_clear; + unsigned long long m_start_byte; + unsigned long long m_end_byte; + unsigned int m_content_length; + }; +}} // azure::storage_lite diff --git a/include/common.h b/include/common.h new file mode 100644 index 0000000..15f3efc --- /dev/null +++ b/include/common.h @@ -0,0 +1,124 @@ +#pragma once + +#include +#include +#include +#include + +namespace azure { namespace storage_lite { + + enum class lease_status + { + locked, + unlocked + }; + + enum class lease_state + { + available, + leased, + expired, + breaking, + broken + }; + + enum class lease_duration + { + none, + infinite, + fixed + }; + + enum class page_write + { + update, + clear + }; + + enum class payload_format + { + json_fullmetadata, + json_nometadata + }; + + // TODO: use the iterator for returned blob response. + //class my_iterator_base : public std::iterator> + //{ + //public: + // virtual bool pass_end() const = 0; + // virtual value_type operator*() = 0; + // virtual void operator++() = 0; + //}; + + //class iterator_base : public std::iterator> + //{ + //public: + // iterator_base(std::shared_ptr iterator) : m_iterator(iterator) {} + + // bool operator!=(const iterator_base &other) const { + // if (m_iterator->pass_end() && other.m_iterator->pass_end()) + // return false; + // return true; + // } + + // value_type operator*() { + // return **m_iterator; + // } + + // iterator_base &operator++() { + // ++(*m_iterator); + // return *this; + // } + + //private: + // std::shared_ptr m_iterator; + //}; + + //class A + //{ + //public: + // virtual iterator_base begin() = 0; + // virtual iterator_base end() = 0; + //}; + + //class my_iterator : public my_iterator_base + //{ + //public: + // my_iterator(std::map::iterator i, std::map::iterator e) : it(i), end_it(e) {} + + // void operator++() override + // { + // ++it; + // } + + // value_type operator*() override + // { + // return std::make_pair(it->first, it->second); + // } + + // bool pass_end() const override + // { + // return it == end_it; + // } + + //private: + // std::map::iterator it; + // std::map::iterator end_it; + //}; + + //class B : public A + //{ + //public: + // iterator_base begin() override + // { + // return iterator_base(std::make_shared(m.begin(), m.end())); + // } + + // iterator_base end() override + // { + // return iterator_base(std::make_shared(m.end(), m.end())); + // } + // std::map m; + //}; + +}} // azure::storage_lite diff --git a/include/constants.dat b/include/constants.dat new file mode 100644 index 0000000..602b96c --- /dev/null +++ b/include/constants.dat @@ -0,0 +1,102 @@ +DAT(http_delete, "DELETE") +DAT(http_get, "GET") +DAT(http_head, "HEAD") +DAT(http_post, "POST") +DAT(http_put, "PUT") + +DAT(default_endpoint_suffix, ".core.windows.net") + +DAT(query_api_version, "api-version") +DAT(query_blockid, "blockid") +DAT(query_blocklisttype, "blocklisttype") +DAT(query_blocklisttype_all, "all") +DAT(query_blocklisttype_committed, "committed") +DAT(query_blocklisttype_uncommitted, "uncommitted") +DAT(query_comp, "comp") +DAT(query_comp_appendblock, "appendblock") +DAT(query_comp_block, "block") +DAT(query_comp_blocklist, "blocklist") +DAT(query_comp_list, "list") +DAT(query_comp_metadata, "metadata") +DAT(query_comp_page, "page") +DAT(query_comp_pagelist, "pagelist") +DAT(query_delimiter, "delimiter") +DAT(query_include, "include") +DAT(query_include_copy, "copy") +DAT(query_include_metadata, "metadata") +DAT(query_include_snapshots, "snapshots") +DAT(query_include_uncommittedblobs, "uncommittedblobs") +DAT(query_marker, "marker") +DAT(query_maxresults, "maxresults") +DAT(query_prefix, "prefix") +DAT(query_restype, "restype") +DAT(query_restype_container, "container") +DAT(query_snapshot, "snapshot") +DAT(query_timeout, "timeout") + +DAT(header_accept, "Accept") +DAT(header_authorization, "Authorization") +DAT(header_cache_control, "Cache-Control") +DAT(header_content_disposition, "Content-Disposition") +DAT(header_content_encoding, "Content-Encoding") +DAT(header_content_range, "Content-Range") +DAT(header_content_language, "Content-Language") +DAT(header_content_length, "Content-Length") +DAT(header_content_md5, "Content-MD5") +DAT(header_content_type, "Content-Type") +DAT(header_etag, "ETag") +DAT(header_if_match, "If-Match") +DAT(header_if_modified_since, "If-Modified-Since") +DAT(header_if_none_match, "If-None-Match") +DAT(header_if_unmodified_since, "If-Unmodified-Since") +DAT(header_last_modified, "Last-Modified") +DAT(header_origin, "Origin") +DAT(header_user_agent, "User-Agent") + +DAT(header_ms_blob_cache_control, "x-ms-blob-cache_control") +DAT(header_ms_blob_condition_appendpos, "x-ms-blob-condition_appendpos") +DAT(header_ms_blob_condition_maxsize, "x-ms-blob-condition_maxsize") +DAT(header_ms_blob_content_disposition, "x-ms-blob-content-disposition") +DAT(header_ms_blob_content_encoding, "x-ms-blob-content-encoding") +DAT(header_ms_blob_content_language, "x-ms-blob-content-language") +DAT(header_ms_blob_content_length, "x-ms-blob-content-length") +DAT(header_ms_blob_content_md5, "x-ms-blob-content-md5") +DAT(header_ms_blob_content_type, "x-ms-blob-content-type") +DAT(header_ms_blob_public_access, "x-ms-blob-public_access") +DAT(header_ms_blob_sequence_number, "x-ms-blob-sequence-number") +DAT(header_ms_blob_type, "x-ms-blob-type") +DAT(header_ms_client_request_id, "x-ms-client-request-id") +DAT(header_ms_copy_source, "x-ms-copy-source") +DAT(header_ms_copy_status, "x-ms-copy-status") +DAT(header_ms_date, "x-ms-date") +DAT(header_ms_delete_snapshots, "x-ms-delete-snapshots") +DAT(header_ms_if_sequence_number_lt, "x-ms-if-sequence-number-lt") +DAT(header_ms_if_sequence_number_le, "x-ms-if-sequence-number-le") +DAT(header_ms_if_sequence_number_eq, "x-ms-if-sequence-number-eq") +DAT(header_ms_lease_id, "x-ms-lease-id") +DAT(header_ms_page_write, "x-ms-page-write") +DAT(header_ms_range, "x-ms-range") +DAT(header_ms_range_get_content_md5, "x-ms-range-get-content-md5") +DAT(header_ms_version, "x-ms-version") + +DAT(header_ms_meta_prefix, "x-ms-meta-") + +DAT(header_value_blob_public_access_blob, "blob") +DAT(header_value_blob_public_access_container, "container") +DAT(header_value_blob_type_blockblob, "BlockBlob") +DAT(header_value_blob_type_pageblob, "PageBlob") +DAT(header_value_blob_type_appendblob, "AppendBlob") +DAT(header_value_delete_snapshots_include, "include") +DAT(header_value_delete_snapshots_only, "only") +DAT(header_value_page_write_update, "update") +DAT(header_value_page_write_clear, "clear") +DAT(header_value_payload_format_nometadata, "application/json;odata=nometadata") +DAT(header_value_payload_format_fullmetadata, "application/json;odata=fullmetadata") +DAT(header_value_storage_version, "2017-04-17") + +DAT(header_value_user_agent, "azure-storage-cpplite/0.1.0") + +DAT(date_format_rfc_1123, "%a, %d %b %Y %H:%M:%S GMT") +DAT(date_format_iso_8601, "%Y-%m-%dT%H:%M:%SZ") + +DAT(code_request_range_not_satisfiable, "416") diff --git a/include/constants.h b/include/constants.h new file mode 100644 index 0000000..178a8b3 --- /dev/null +++ b/include/constants.h @@ -0,0 +1,9 @@ +#pragma once + +namespace azure { namespace storage_lite { namespace constants { + +#define DAT(x, y) extern const char *x; const int x ## _size{ sizeof(y) / sizeof(char) - 1 }; +#include "constants.dat" +#undef DAT + +}}} // azure::storage_lite::constants \ No newline at end of file diff --git a/include/copy_blob_request_base.h b/include/copy_blob_request_base.h new file mode 100644 index 0000000..444c4d0 --- /dev/null +++ b/include/copy_blob_request_base.h @@ -0,0 +1,26 @@ +#pragma once + +#include + +#include "storage_EXPORTS.h" + +#include "http_base.h" +#include "storage_account.h" +#include "storage_request_base.h" + +namespace azure { namespace storage_lite { + + class copy_blob_request_base : public blob_request_base + { + public: + + virtual std::string container() const = 0; + virtual std::string blob() const = 0; + + virtual std::string destContainer() const = 0; + virtual std::string destBlob() const = 0; + + AZURE_STORAGE_API void build_request(const storage_account &a, http_base &h) const override; + }; + +}} // azure::storage_lite diff --git a/include/create_container_request_base.h b/include/create_container_request_base.h new file mode 100644 index 0000000..0bdec3b --- /dev/null +++ b/include/create_container_request_base.h @@ -0,0 +1,35 @@ +#pragma once + +#include +#include + +#include "storage_EXPORTS.h" + +#include "http_base.h" +#include "storage_account.h" +#include "storage_request_base.h" + +namespace azure { namespace storage_lite { + + class create_container_request_base : public blob_request_base + { + public: + enum class blob_public_access + { + unspecified, + container, + blob + }; + + virtual std::string container() const = 0; + + virtual blob_public_access ms_blob_public_access() const + { + return blob_public_access::unspecified; + } + + AZURE_STORAGE_API void build_request(const storage_account &a, http_base &h) const override; + }; + +}} // azure::storage_lite + diff --git a/include/delete_blob_request_base.h b/include/delete_blob_request_base.h new file mode 100644 index 0000000..b03454b --- /dev/null +++ b/include/delete_blob_request_base.h @@ -0,0 +1,32 @@ +#pragma once + +#include + +#include "storage_EXPORTS.h" + +#include "http_base.h" +#include "storage_account.h" +#include "storage_request_base.h" + +namespace azure { namespace storage_lite { + + class delete_blob_request_base : public blob_request_base + { + public: + enum class delete_snapshots + { + unspecified, + include, + only + }; + + virtual std::string container() const = 0; + virtual std::string blob() const = 0; + + virtual std::string snapshot() const { return std::string(); } + virtual delete_snapshots ms_delete_snapshots() const { return delete_snapshots::unspecified; } + + AZURE_STORAGE_API void build_request(const storage_account &a, http_base &h) const override; + }; + +}} // azure::storage_lite \ No newline at end of file diff --git a/include/delete_container_request_base.h b/include/delete_container_request_base.h new file mode 100644 index 0000000..7ef4911 --- /dev/null +++ b/include/delete_container_request_base.h @@ -0,0 +1,22 @@ +#pragma once + +#include +#include + +#include "storage_EXPORTS.h" + +#include "http_base.h" +#include "storage_account.h" +#include "storage_request_base.h" + +namespace azure { namespace storage_lite { + + class delete_container_request_base : public blob_request_base + { + public: + virtual std::string container() const = 0; + + AZURE_STORAGE_API void build_request(const storage_account &a, http_base &h) const override; + }; + +}} // azure::storage_lite \ No newline at end of file diff --git a/include/executor.h b/include/executor.h new file mode 100644 index 0000000..3d049f6 --- /dev/null +++ b/include/executor.h @@ -0,0 +1,207 @@ +#pragma once + +#include +#include +#include +#include +#include + +#include "storage_EXPORTS.h" + +#include "common.h" +#include "storage_outcome.h" +#include "storage_account.h" +#include "http_base.h" +#include "xml_parser_base.h" +#include "retry.h" +#include "utility.h" + +namespace azure { namespace storage_lite { + + class executor_context + { + public: + executor_context(std::shared_ptr xml_parser, std::shared_ptr retry) + : m_xml_parser(xml_parser), + m_retry_policy(retry) {} + + std::shared_ptr xml_parser() const + { + return m_xml_parser; + } + + std::shared_ptr retry_policy() const + { + return m_retry_policy; + } + + private: + std::shared_ptr m_xml_parser; + std::shared_ptr m_retry_policy; + }; + + template + class async_executor + { + public: + static void submit_request(std::promise> &promise, const storage_account &a, const storage_request_base &r, http_base &h, const executor_context &context, retry_context &retry) + { + h.set_error_stream([](http_base::http_code) { return true; }, storage_iostream::create_storage_stream()); + r.build_request(a, h); + + retry_info info = context.retry_policy()->evaluate(retry); + if (info.should_retry()) { + h.submit([&promise, &a, &r, &h, &context, &retry](http_base::http_code result, storage_istream s, CURLcode code) { + std::string str(std::istreambuf_iterator(s.istream()), std::istreambuf_iterator()); + if (code != CURLE_OK || unsuccessful(result)) { + promise.set_value(storage_outcome(context.xml_parser()->parse_storage_error(str))); + retry.add_result(code == CURLE_OK ? result : 503); + h.reset_input_stream(); + h.reset_output_stream(); + async_executor::submit_request(promise, a, r, h, context, retry); + } + else { + promise.set_value(storage_outcome(context.xml_parser()->parse_response(str))); + } + }, info.interval()); + } + } + + static void submit_helper( + std::shared_ptr>> promise, + std::shared_ptr> outcome, + std::shared_ptr account, + std::shared_ptr request, + std::shared_ptr http, + std::shared_ptr context, + std::shared_ptr retry) + { + http->set_error_stream([](http_base::http_code) { return true; }, storage_iostream::create_storage_stream()); + request->build_request(*account, *http); + + retry_info info = context->retry_policy()->evaluate(*retry); + if (info.should_retry()) + { + http->submit([promise, outcome, account, request, http, context, retry](http_base::http_code result, storage_istream s, CURLcode code) + { + std::string str(std::istreambuf_iterator(s.istream()), std::istreambuf_iterator()); + if (code != CURLE_OK || unsuccessful(result)) + { + auto error = context->xml_parser()->parse_storage_error(str); + error.code = std::to_string(result); + *outcome = storage_outcome(error); + retry->add_result(code == CURLE_OK ? result: 503); + http->reset_input_stream(); + http->reset_output_stream(); + async_executor::submit_helper(promise, outcome, account, request, http, context, retry); + } + else + { + *outcome = storage_outcome(context->xml_parser()->parse_response(str)); + promise->set_value(*outcome); + } + }, info.interval()); + } + else + { + promise->set_value(*outcome); + } + } + + static std::future> submit( + std::shared_ptr account, + std::shared_ptr request, + std::shared_ptr http, + std::shared_ptr context) + { + auto retry = std::make_shared(); + auto outcome = std::make_shared>(); + auto promise = std::make_shared>>(); + async_executor::submit_helper(promise, outcome, account, request, http, context, retry); + return promise->get_future(); + } + }; + + template<> + class async_executor { + public: + static void submit_request(std::promise> &promise, const storage_account &a, const storage_request_base &r, http_base &h, const executor_context &context, retry_context &retry) + { + h.set_error_stream(unsuccessful, storage_iostream::create_storage_stream()); + r.build_request(a, h); + + retry_info info = context.retry_policy()->evaluate(retry); + if (info.should_retry()) { + h.submit([&promise, &a, &r, &h, &context, &retry](http_base::http_code result, storage_istream s, CURLcode code) { + std::string str(std::istreambuf_iterator(s.istream()), std::istreambuf_iterator()); + if (code != CURLE_OK || unsuccessful(result)) { + promise.set_value(storage_outcome(context.xml_parser()->parse_storage_error(str))); + retry.add_result(code == CURLE_OK ? result : 503); + h.reset_input_stream(); + h.reset_output_stream(); + async_executor::submit_request(promise, a, r, h, context, retry); + } + else { + promise.set_value(storage_outcome()); + } + }, info.interval()); + } + } + + static void submit_helper( + std::shared_ptr>> promise, + std::shared_ptr> outcome, + std::shared_ptr account, + std::shared_ptr request, + std::shared_ptr http, + std::shared_ptr context, + std::shared_ptr retry) + { + http->reset(); + http->set_error_stream(unsuccessful, storage_iostream::create_storage_stream()); + request->build_request(*account, *http); + + retry_info info = context->retry_policy()->evaluate(*retry); + if (info.should_retry()) + { + http->submit([promise, outcome, account, request, http, context, retry](http_base::http_code result, storage_istream s, CURLcode code) + { + std::string str(std::istreambuf_iterator(s.istream()), std::istreambuf_iterator()); + if (code != CURLE_OK || unsuccessful(result)) + { + auto error = context->xml_parser()->parse_storage_error(str); + error.code = std::to_string(result); + *outcome = storage_outcome(error); + retry->add_result(code == CURLE_OK ? result: 503); + http->reset_input_stream(); + http->reset_output_stream(); + async_executor::submit_helper(promise, outcome, account, request, http, context, retry); + } + else + { + *outcome = storage_outcome(); + promise->set_value(*outcome); + } + }, info.interval()); + } + else + { + promise->set_value(*outcome); + } + } + + static std::future> submit( + std::shared_ptr account, + std::shared_ptr request, + std::shared_ptr http, + std::shared_ptr context) + { + auto retry = std::make_shared(); + auto outcome = std::make_shared>(); + auto promise = std::make_shared>>(); + async_executor::submit_helper(promise, outcome, account, request, http, context, retry); + return promise->get_future(); + } + }; + } +} diff --git a/include/get_blob_property_request_base.h b/include/get_blob_property_request_base.h new file mode 100644 index 0000000..fb7dbb3 --- /dev/null +++ b/include/get_blob_property_request_base.h @@ -0,0 +1,65 @@ +#pragma once + +#include + +#include "storage_EXPORTS.h" + +#include "http_base.h" +#include "storage_account.h" +#include "storage_request_base.h" +#include "get_blob_request_base.h" + +namespace azure { namespace storage_lite { + + class get_blob_property_request_base : public blob_request_base + { + public: + virtual std::string container() const = 0; + virtual std::string blob() const = 0; + + virtual std::string snapshot() const { return std::string(); } + + AZURE_STORAGE_API void build_request(const storage_account &a, http_base &h) const override; + }; + + class blob_property + { + public: + blob_property(bool valid) + :last_modified{time(NULL)}, + m_valid(valid) + { + } + + void set_valid(bool valid) + { + m_valid = valid; + } + + bool valid() + { + return m_valid; + } + + std::string cache_control; + std::string content_disposition; + std::string content_encoding; + std::string content_language; + unsigned long long size; + std::string content_md5; + std::string content_type; + std::string etag; + std::vector> metadata; + std::string copy_status; + time_t last_modified; + // TODO: support lease and blob_type + // blob_type m_type; + // azure::storage::lease_status m_lease_status; + // azure::storage::lease_state m_lease_state; + // azure::storage::lease_duration m_lease_duration; + + private: + blob_property() {} + bool m_valid; + }; +}} // azure::storage_lite diff --git a/include/get_blob_request_base.h b/include/get_blob_request_base.h new file mode 100644 index 0000000..53fd5af --- /dev/null +++ b/include/get_blob_request_base.h @@ -0,0 +1,42 @@ +#pragma once + +#include + +#include "storage_EXPORTS.h" + +#include "http_base.h" +#include "storage_account.h" +#include "storage_request_base.h" + +namespace azure { namespace storage_lite { + + class get_blob_request_base : public blob_request_base + { + public: + virtual std::string container() const = 0; + virtual std::string blob() const = 0; + + virtual std::string snapshot() const { return std::string(); } + virtual unsigned long long start_byte() const { return 0; } + virtual unsigned long long end_byte() const { return 0; } + virtual std::string origin() const { return std::string(); } + virtual bool ms_range_get_content_md5() const { return false; } + + AZURE_STORAGE_API void build_request(const storage_account &a, http_base &h) const override; + }; + + class chunk_property + { + public: + chunk_property() + :totalSize{0}, + size{0}, + last_modified{0} //returns 1970 + { + } + long long totalSize; + unsigned long long size; + time_t last_modified; + std::string etag; + }; +}} // azure::storage_lite diff --git a/include/get_block_list_request_base.h b/include/get_block_list_request_base.h new file mode 100644 index 0000000..a085355 --- /dev/null +++ b/include/get_block_list_request_base.h @@ -0,0 +1,46 @@ +#pragma once + +#include + +#include "storage_EXPORTS.h" + +#include "http_base.h" +#include "storage_account.h" +#include "storage_request_base.h" + +namespace azure { namespace storage_lite { + + class get_block_list_request_base : public blob_request_base + { + public: + enum class blocklisttypes + { + committed, + uncommitted, + all + }; + + virtual std::string container() const = 0; + virtual std::string blob() const = 0; + + virtual std::string snapshot() const { return std::string(); } + virtual blocklisttypes blocklisttype() const { return blocklisttypes::all; } + + AZURE_STORAGE_API void build_request(const storage_account &a, http_base &h) const override; + }; + + class get_block_list_item + { + public: + std::string name; + unsigned long long size; + }; + + class get_block_list_response + { + public: + std::vector committed; + std::vector uncommitted; + }; + +}} // azure::storage_lite diff --git a/include/get_container_property_request_base.h b/include/get_container_property_request_base.h new file mode 100644 index 0000000..4ec2ac6 --- /dev/null +++ b/include/get_container_property_request_base.h @@ -0,0 +1,46 @@ +#pragma once + +#include + +#include "storage_EXPORTS.h" + +#include "http_base.h" +#include "storage_account.h" +#include "storage_request_base.h" + +namespace azure { namespace storage_lite { + + class get_container_property_request_base : public blob_request_base + { + public: + virtual std::string container() const = 0; + + AZURE_STORAGE_API void build_request(const storage_account &a, http_base &h) const override; + }; + + class container_property + { + public: + container_property(bool valid) + :m_valid(valid) + { + } + + void set_valid(bool valid) + { + m_valid = valid; + } + + bool valid() + { + return m_valid; + } + + std::string etag; + std::vector> metadata; + + private: + container_property() {} + bool m_valid; + }; +}} // azure::storage_lite diff --git a/include/get_page_ranges_request_base.h b/include/get_page_ranges_request_base.h new file mode 100644 index 0000000..a465cf2 --- /dev/null +++ b/include/get_page_ranges_request_base.h @@ -0,0 +1,40 @@ +#pragma once + +#include + +#include "storage_EXPORTS.h" + +#include "http_base.h" +#include "storage_account.h" +#include "storage_request_base.h" + +namespace azure { namespace storage_lite { + + class get_page_ranges_request_base : public blob_request_base + { + public: + virtual std::string container() const = 0; + virtual std::string blob() const = 0; + + virtual unsigned long long start_byte() const { return 0; } + virtual unsigned long long end_byte() const { return 0; } + + virtual std::string snapshot() const { return std::string(); } + + AZURE_STORAGE_API void build_request(const storage_account &a, http_base &h) const override; + }; + + class get_page_ranges_item + { + public: + unsigned long long start; + unsigned long long end; + }; + + class get_page_ranges_response + { + public: + std::vector pagelist; + }; + +}} // azure::storage_lite diff --git a/include/hash.h b/include/hash.h new file mode 100644 index 0000000..5887917 --- /dev/null +++ b/include/hash.h @@ -0,0 +1,67 @@ +#pragma once + +#include +#include + +#ifdef _WIN32 +#include +#include +#else +#ifdef USE_OPENSSL +#include +#else +#include +#include +#endif +#define SHA256_DIGEST_LENGTH 32 +#endif + +#include "storage_EXPORTS.h" + +namespace azure { namespace storage_lite { + +#ifdef _WIN32 + class hash_algorithm_base {}; + + class hmac_sha256_hash_algorithm : public hash_algorithm_base + { + BCRYPT_ALG_HANDLE _algorithm_handle; + public: + BCRYPT_ALG_HANDLE handle() { return _algorithm_handle; } + + hmac_sha256_hash_algorithm() + { + NTSTATUS status = BCryptOpenAlgorithmProvider(&_algorithm_handle, BCRYPT_SHA256_ALGORITHM /* BCRYPT_MD5_ALGORITHM */, NULL, BCRYPT_ALG_HANDLE_HMAC_FLAG /* 0 */); + if (status != 0) { + throw std::system_error(status, std::system_category()); + } + } + + ~hmac_sha256_hash_algorithm() + { + BCryptCloseAlgorithmProvider(_algorithm_handle, 0); + } + }; + + template + class hash_provider_base + { + static std::string hash_impl(const std::string &input, const std::vector &key) = delete; + public: + static std::string hash(const std::string &input, const std::vector &key + { + return Hash_Provider::hash_impl(input, key); + } + }; + + class hmac_sha256_hash_provider : public hash_provider_base + { + static hmac_sha256_hash_algorithm _algorithm; + public: + AZURE_STORAGE_API static std::string hash_impl(const std::string &input, const std::vector &key); + }; +#else + std::string hash(const std::string &to_sign, const std::vector &key); +#endif + +}} // azure::storage_lite diff --git a/include/http/libcurl_http_client.h b/include/http/libcurl_http_client.h new file mode 100644 index 0000000..68d3b02 --- /dev/null +++ b/include/http/libcurl_http_client.h @@ -0,0 +1,275 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "storage_EXPORTS.h" + +#include "http_base.h" + +namespace azure { namespace storage_lite { + + class CurlEasyClient; + + class CurlEasyRequest : public http_base + { + + using REQUEST_TYPE = CurlEasyRequest; + + public: + AZURE_STORAGE_API CurlEasyRequest(std::shared_ptr client, CURL *h); + + AZURE_STORAGE_API ~CurlEasyRequest(); + + void set_url(const std::string &url) override + { + m_url = url; + } + + std::string get_url() const override + { + return m_url; + } + + void set_method(http_method method) override + { + m_method = method; + } + + http_method get_method() const override + { + return m_method; + } + + void add_header(const std::string &name, const std::string &value) override + { + std::string header(name); + header.append(": ").append(value); + m_slist = curl_slist_append(m_slist, header.data()); + if (name == "Content-Length") { + unsigned int l; + std::istringstream iss(value); + iss >> l; + curl_easy_setopt(m_curl, CURLOPT_INFILESIZE, l); + } + } + + std::string get_header(const std::string &name) const override + { + auto iter = m_headers.find(name); + if (iter != m_headers.end()) + { + return iter->second; + } + else + { + return ""; + } + } + const std::map& get_headers() const override + { + return m_headers; + } + + AZURE_STORAGE_API CURLcode perform() override; + + void submit(std::function cb, std::chrono::seconds interval) override + { + std::this_thread::sleep_for(interval); + const auto curlCode = perform(); + cb(m_code, m_error_stream, curlCode); + } + + void reset() override + { + m_headers.clear(); + curl_slist_free_all(m_slist); + m_slist = NULL; + } + + http_code status_code() const override + { + return m_code; + } + + void set_output_stream(storage_ostream s) override + { + m_output_stream = s; + check_code(curl_easy_setopt(m_curl, CURLOPT_WRITEFUNCTION, write)); + check_code(curl_easy_setopt(m_curl, CURLOPT_WRITEDATA, this)); + } + + void set_error_stream(std::function f, storage_iostream s) override + { + m_switch_error_callback = f; + m_error_stream = s; + } + + void set_input_stream(storage_istream s) override + { + m_input_stream = s; + check_code(curl_easy_setopt(m_curl, CURLOPT_READFUNCTION, read)); + check_code(curl_easy_setopt(m_curl, CURLOPT_READDATA, this)); + } + + void reset_input_stream() override + { + m_input_stream.reset(); + } + + void reset_output_stream() override + { + m_output_stream.reset(); + } + + storage_ostream get_output_stream() const override + { + return m_output_stream; + } + + storage_iostream get_error_stream() const override + { + return m_error_stream; + } + + storage_istream get_input_stream() const override + { + return m_input_stream; + } + + void set_absolute_timeout(long long timeout) override + { + check_code(curl_easy_setopt(m_curl, CURLOPT_TIMEOUT, timeout)); // Absolute timeout + + // For the moment, we are only using one type of timeout per operation, so we clear the other one, in case it was set for this handle by a prior operation: + check_code(curl_easy_setopt(m_curl, CURLOPT_LOW_SPEED_TIME, 0L)); + check_code(curl_easy_setopt(m_curl, CURLOPT_LOW_SPEED_LIMIT, 0L)); + } + + void set_data_rate_timeout() override + { + // If the download speed is less than 17KB/sec for more than a minute, timout. This time was selected because it should ensure that downloading each megabyte take no more than a minute. + check_code(curl_easy_setopt(m_curl, CURLOPT_LOW_SPEED_TIME, 60L)); + check_code(curl_easy_setopt(m_curl, CURLOPT_LOW_SPEED_LIMIT, 1024L * 17L)); + + // For the moment, we are only using one type of timeout per operation, so we clear the other one, in case it was set for this handle by a prior operation: + check_code(curl_easy_setopt(m_curl, CURLOPT_TIMEOUT, 0L)); + } + + private: + std::shared_ptr m_client; + CURL *m_curl; + curl_slist *m_slist; + + http_method m_method; + std::string m_url; + storage_istream m_input_stream; + storage_ostream m_output_stream; + storage_iostream m_error_stream; + std::function m_switch_error_callback; + + http_code m_code; + std::map m_headers; + + AZURE_STORAGE_API static size_t header_callback(char *buffer, size_t size, size_t nitems, void *userdata); + + static size_t write(char *buffer, size_t size, size_t nitems, void *userdata) + { + REQUEST_TYPE *p = static_cast(userdata); + p->m_output_stream.ostream().write(buffer, size * nitems); + return size * nitems; + } + + static size_t error(char *buffer, size_t size, size_t nitems, void *userdata) + { + REQUEST_TYPE *p = static_cast(userdata); + p->m_error_stream.ostream().write(buffer, size * nitems); + return size * nitems; + } + + static size_t read(char *buffer, size_t size, size_t nitems, void *userdata) + { + REQUEST_TYPE *p = static_cast(userdata); + auto &s = p->m_input_stream.istream(); + + auto cur = s.tellg(); + s.seekg(0, std::ios_base::end); + auto end = s.tellg(); + s.seekg(cur); + + auto actual_size = std::min(static_cast(end - cur), size * nitems); + s.read(buffer, actual_size); + return actual_size; + } + + static void check_code(CURLcode code, std::string = std::string()) + { + if (code == CURLE_OK) + { + errno = 0; // CURL sometimes sets errno internally, if everything was ok we should reset it to zero. + } + } + }; + + class CurlEasyClient : public std::enable_shared_from_this + { + public: + CurlEasyClient(int size) : m_size(size) + { + curl_global_init(CURL_GLOBAL_DEFAULT); + for (int i = 0; i < m_size; i++) { + CURL *h = curl_easy_init(); + m_handles.push(h); + } + } + + ~CurlEasyClient() { + while (!m_handles.empty()) + { + curl_easy_cleanup(m_handles.front()); + m_handles.pop(); + } + curl_global_cleanup(); + } + + int size() + { + return m_size; + } + + std::shared_ptr get_handle() + { + std::unique_lock lk(m_handles_mutex); + m_cv.wait(lk, [this]() { return !m_handles.empty(); }); + auto res = std::make_shared(shared_from_this(), m_handles.front()); + m_handles.pop(); + return res; + } + + void release_handle(CURL *h) + { + std::lock_guard lg(m_handles_mutex); + m_handles.push(h); + m_cv.notify_one(); + } + + private: + int m_size; + std::queue m_handles; + std::mutex m_handles_mutex; + std::condition_variable m_cv; + }; + +}} // azure::storage_lite diff --git a/include/http_base.h b/include/http_base.h new file mode 100644 index 0000000..88885fd --- /dev/null +++ b/include/http_base.h @@ -0,0 +1,69 @@ +#pragma once + +#include +#include +#include +#include +#include + +#include "storage_stream.h" + +namespace azure { namespace storage_lite { + + class http_base + { + public: + enum class http_method + { + del, + get, + head, + post, + put + }; + + using http_code = int; + + virtual void set_method(http_method method) = 0; + + virtual http_method get_method() const = 0; + + virtual void set_url(const std::string &url) = 0; + + virtual std::string get_url() const = 0; + + virtual void add_header(const std::string &name, const std::string &value) = 0; + + virtual std::string get_header(const std::string &name) const = 0; + virtual const std::map& get_headers() const = 0; + + virtual CURLcode perform() = 0; + + virtual void submit(std::function cb, std::chrono::seconds interval) = 0; + + virtual void reset() = 0; + + virtual http_code status_code() const = 0; + + virtual void set_input_stream(storage_istream s) = 0; + + virtual void reset_input_stream() = 0; + + virtual void reset_output_stream() = 0; + + virtual void set_output_stream(storage_ostream s) = 0; + + virtual void set_error_stream(std::function f, storage_iostream s) = 0; + + virtual storage_istream get_input_stream() const = 0; + + virtual storage_ostream get_output_stream() const = 0; + + virtual storage_iostream get_error_stream() const = 0; + + virtual void set_absolute_timeout(long long timeout) = 0; + + virtual void set_data_rate_timeout() = 0; + }; + +}} // azure::storage_lite diff --git a/include/list_blobs_request_base.h b/include/list_blobs_request_base.h new file mode 100644 index 0000000..bbcb228 --- /dev/null +++ b/include/list_blobs_request_base.h @@ -0,0 +1,108 @@ +#pragma once + +#include +#include +#include + +#include "storage_EXPORTS.h" + +#include "common.h" +#include "http_base.h" +#include "storage_account.h" +#include "storage_request_base.h" + +namespace azure { namespace storage_lite { + + class list_blobs_request_base : public blob_request_base + { + public: + enum include + { + unspecifies = 0x0, + snapshots = 0x1, + metadata = 0x2, + uncommittedblobs = 0x4, + copy = 0x8 + }; + + virtual std::string container() const = 0; + virtual std::string prefix() const { return std::string(); } + virtual std::string delimiter() const { return std::string(); } + virtual std::string marker() const { return std::string(); } + virtual int maxresults() const { return 0; } + virtual include includes() const { return include::unspecifies; } + + AZURE_STORAGE_API void build_request(const storage_account &a, http_base &h) const override; + }; + + class list_blobs_item + { + public: + std::string name; + std::string snapshot; + std::string last_modified; + std::string etag; + unsigned long long content_length; + std::string content_encoding; + std::string content_type; + std::string content_md5; + std::string content_language; + std::string cache_control; + lease_status status; + lease_state state; + lease_duration duration; + }; + + class list_blobs_response + { + public: + std::string ms_request_id; + std::vector blobs; + std::string next_marker; + }; + + + class list_blobs_segmented_request_base : public blob_request_base + { + public: + + virtual std::string container() const = 0; + virtual std::string prefix() const { return std::string(); } + virtual std::string delimiter() const { return std::string(); } + virtual std::string marker() const { return std::string(); } + virtual int maxresults() const { return 0; } + virtual list_blobs_request_base::include includes() const { return list_blobs_request_base::include::unspecifies; } + + AZURE_STORAGE_API void build_request(const storage_account &a, http_base &h) const override; + }; + + class list_blobs_segmented_item + { + public: + std::string name; + std::string snapshot; + std::string last_modified; + std::string etag; + unsigned long long content_length; + std::string content_encoding; + std::string content_type; + std::string content_md5; + std::string content_language; + std::string cache_control; + lease_status status; + lease_state state; + lease_duration duration; + std::vector> metadata; + bool is_directory; + }; + + class list_blobs_segmented_response + { + public: + std::string ms_request_id; + std::vector blobs; + std::string next_marker; + }; + +}} // azure::storage_lite + diff --git a/include/list_containers_request_base.h b/include/list_containers_request_base.h new file mode 100644 index 0000000..8561e21 --- /dev/null +++ b/include/list_containers_request_base.h @@ -0,0 +1,47 @@ +#pragma once + +#include +#include +#include + +#include "storage_EXPORTS.h" + +#include "common.h" +#include "http_base.h" +#include "storage_account.h" +#include "storage_request_base.h" + +namespace azure { namespace storage_lite { + + class list_containers_request_base : public blob_request_base + { + public: + virtual std::string prefix() const { return std::string(); } + virtual std::string marker() const { return std::string(); } + virtual int maxresults() const { return 0; } + virtual bool include_metadata() const { return false; } + + AZURE_STORAGE_API void build_request(const storage_account &a, http_base &h) const override; + }; + + class list_containers_item + { + public: + std::string name; + std::string last_modified; + std::string etag; + lease_status status; + lease_state state; + lease_duration duration; + }; + + class list_constainers_segmented_response + { + public: + std::string ms_request_id; + std::vector containers; + std::string next_marker; + }; + +}} // azure::storage_lite + diff --git a/include/put_blob_request_base.h b/include/put_blob_request_base.h new file mode 100644 index 0000000..3ae7cf4 --- /dev/null +++ b/include/put_blob_request_base.h @@ -0,0 +1,50 @@ +#pragma once + +#include +#include + +#include "storage_EXPORTS.h" + +#include "http_base.h" +#include "storage_account.h" +#include "storage_request_base.h" + +namespace azure { namespace storage_lite { + + class put_blob_request_base : public blob_request_base + { + public: + enum class blob_type + { + block_blob, + page_blob, + append_blob + }; + + virtual std::string container() const = 0; + virtual std::string blob() const = 0; + virtual std::vector> metadata() const = 0; + + virtual std::string content_encoding() const { return std::string(); } + virtual std::string content_language() const { return std::string(); } + virtual unsigned int content_length() const = 0; + virtual std::string content_md5() const { return std::string(); } + virtual std::string content_type() const { return std::string(); } + + virtual std::string origin() const { return std::string(); } + virtual std::string cache_control() const { return std::string(); } + + virtual std::string ms_blob_cache_control() const { return std::string(); } + virtual std::string ms_blob_content_disposition() const { return std::string(); } + virtual std::string ms_blob_content_encoding() const { return std::string(); } + virtual std::string ms_blob_content_language() const { return std::string(); } + virtual unsigned long long ms_blob_content_length() const { return 0; } + virtual std::string ms_blob_content_md5() const { return std::string(); } + virtual std::string ms_blob_content_type() const { return std::string(); } + virtual unsigned long long ms_blob_sequence_number() const { return 0; } + virtual blob_type ms_blob_type() const = 0; + + AZURE_STORAGE_API void build_request(const storage_account &a, http_base &h) const override; + }; + +}} diff --git a/include/put_block_list_request_base.h b/include/put_block_list_request_base.h new file mode 100644 index 0000000..ad6aa43 --- /dev/null +++ b/include/put_block_list_request_base.h @@ -0,0 +1,47 @@ +#pragma once + +#include +#include + +#include "storage_EXPORTS.h" + +#include "http_base.h" +#include "storage_account.h" +#include "storage_request_base.h" + +namespace azure { namespace storage_lite { + + class put_block_list_request_base : public blob_request_base + { + public: + enum class block_type { + committed, + uncommitted, + latest + }; + + class block_item { + public: + std::string id; + block_type type; + }; + + virtual std::string container() const = 0; + virtual std::string blob() const = 0; + + virtual std::vector block_list() const = 0; + virtual std::vector> metadata() const = 0; + + virtual std::string content_md5() const { return std::string(); } + + virtual std::string ms_blob_cache_control() const { return std::string(); } + virtual std::string ms_blob_content_disposition() const { return std::string(); } + virtual std::string ms_blob_content_encoding() const { return std::string(); } + virtual std::string ms_blob_content_language() const { return std::string(); } + virtual std::string ms_blob_content_md5() const { return std::string(); } + virtual std::string ms_blob_content_type() const { return std::string(); } + + AZURE_STORAGE_API void build_request(const storage_account &a, http_base &h) const override; + }; + +}} // azure::storage_lite diff --git a/include/put_block_request_base.h b/include/put_block_request_base.h new file mode 100644 index 0000000..262241b --- /dev/null +++ b/include/put_block_request_base.h @@ -0,0 +1,27 @@ +#pragma once + +#include +#include + +#include "storage_EXPORTS.h" + +#include "http_base.h" +#include "storage_account.h" +#include "storage_request_base.h" + +namespace azure { namespace storage_lite { + + class put_block_request_base : public blob_request_base + { + public: + virtual std::string container() const = 0; + virtual std::string blob() const = 0; + virtual std::string blockid() const = 0; + + virtual unsigned int content_length() const = 0; + virtual std::string content_md5() const { return std::string(); } + + AZURE_STORAGE_API void build_request(const storage_account &a, http_base &h) const override; + }; + +}} // azure::storage_lite diff --git a/include/put_page_request_base.h b/include/put_page_request_base.h new file mode 100644 index 0000000..b16486e --- /dev/null +++ b/include/put_page_request_base.h @@ -0,0 +1,39 @@ +#pragma once + +#include +#include + +#include "storage_EXPORTS.h" + +#include "http_base.h" +#include "storage_account.h" +#include "storage_request_base.h" + +namespace azure { namespace storage_lite { + + class put_page_request_base : public blob_request_base + { + public: + enum class page_write + { + update, + clear + }; + + virtual std::string container() const = 0; + virtual std::string blob() const = 0; + + virtual unsigned long long start_byte() const { return 0; } + virtual unsigned long long end_byte() const { return 0; } + virtual page_write ms_page_write() const = 0; + virtual std::string ms_if_sequence_number_le() const { return std::string(); } + virtual std::string ms_if_sequence_number_lt() const { return std::string(); } + virtual std::string ms_if_sequence_number_eq() const { return std::string(); } + + virtual unsigned int content_length() const = 0; + virtual std::string content_md5() const { return std::string(); } + + AZURE_STORAGE_API void build_request(const storage_account &a, http_base &h) const override; + }; + +}} // azure::storage_lite diff --git a/include/retry.h b/include/retry.h new file mode 100644 index 0000000..d976c20 --- /dev/null +++ b/include/retry.h @@ -0,0 +1,100 @@ +#pragma once + +#include +#include +#include + +#include "storage_EXPORTS.h" + +#include "http_base.h" +#include "utility.h" + +namespace azure { namespace storage_lite { + + class retry_info + { + public: + retry_info(bool should_retry, std::chrono::seconds interval) + : m_should_retry(should_retry), + m_interval(interval) {} + + bool should_retry() const + { + return m_should_retry; + } + + std::chrono::seconds interval() const + { + return m_interval; + } + + private: + bool m_should_retry; + std::chrono::seconds m_interval; + }; + + class retry_context + { + public: + retry_context() + : m_numbers(0), + m_result(0) {} + + retry_context(int numbers, http_base::http_code result) + : m_numbers(numbers), + m_result(result) {} + + int numbers() const + { + return m_numbers; + } + + http_base::http_code result() const + { + return m_result; + } + + void add_result(http_base::http_code result) + { + m_result = result; + m_numbers++; + } + + private: + int m_numbers; + http_base::http_code m_result; + }; + + class retry_policy_base + { + public: + virtual retry_info evaluate(const retry_context &context) const = 0; + }; + + class retry_policy : public retry_policy_base + { + public: + retry_info evaluate(const retry_context &context) const override + { + if (context.numbers() == 0) + { + return retry_info(true, std::chrono::seconds(0)); + } + else if (context.numbers() < 26 && can_retry(context.result())) + { + double delay = (pow(1.2, context.numbers()-1)-1); + delay = std::min(delay, 60.0); // Maximum backoff delay of 1 minute + delay *= (((double)rand())/RAND_MAX)/2 + 0.75; + return retry_info(true, std::chrono::seconds((int)delay)); + } + return retry_info(false, std::chrono::seconds(0)); + } + + private: + bool can_retry(http_base::http_code code) const + { + return retryable(code); + } + }; + +}} // azure::storage_lite diff --git a/include/storage_EXPORTS.h b/include/storage_EXPORTS.h new file mode 100644 index 0000000..1976814 --- /dev/null +++ b/include/storage_EXPORTS.h @@ -0,0 +1,7 @@ +#pragma once + +#ifdef _WIN32 +#define AZURE_STORAGE_API __declspec(dllexport) +#else /* ifdef WIN32 */ +#define AZURE_STORAGE_API +#endif diff --git a/include/storage_account.h b/include/storage_account.h new file mode 100644 index 0000000..54788bb --- /dev/null +++ b/include/storage_account.h @@ -0,0 +1,43 @@ +#pragma once + +#include +#include + +#include "storage_EXPORTS.h" + +#include "storage_credential.h" +#include "storage_url.h" + +namespace azure { namespace storage_lite { + + class storage_account + { + public: + enum class service + { + blob, + table, + queue, + file + }; + + AZURE_STORAGE_API storage_account(const std::string &account_name, std::shared_ptr credential, bool use_https = true, const std::string &blob_endpoint = std::string()); + + std::shared_ptr credential() const + { + return m_credential; + } + + AZURE_STORAGE_API storage_url get_url(service service) const; + + private: + std::shared_ptr m_credential; + std::string m_blob_domain; + std::string m_table_domain; + std::string m_queue_domain; + std::string m_file_domain; + + AZURE_STORAGE_API void append_all(const std::string &part); + }; + +}} // azure::storage_lite diff --git a/include/storage_credential.h b/include/storage_credential.h new file mode 100644 index 0000000..e535679 --- /dev/null +++ b/include/storage_credential.h @@ -0,0 +1,76 @@ +#pragma once + +#include +#include + +#include "storage_EXPORTS.h" + +#include "http_base.h" +#include "storage_request_base.h" +#include "storage_url.h" + +namespace azure { namespace storage_lite { + + class storage_credential + { + public: + virtual void sign_request(const storage_request_base &, http_base &, const storage_url &, const storage_headers &) const {} + virtual std::string transform_url(std::string url) const + { + return url; + } + }; + + class shared_key_credential : public storage_credential + { + public: + AZURE_STORAGE_API shared_key_credential(const std::string &account_name, const std::string &account_key); + + AZURE_STORAGE_API shared_key_credential(const std::string &account_name, const std::vector &account_key); + + AZURE_STORAGE_API void sign_request(const storage_request_base &r, http_base &h, const storage_url &url, const storage_headers &headers) const override; + + AZURE_STORAGE_API void sign_request(const table_request_base &r, http_base &h, const storage_url &url, const storage_headers &headers) const; + + const std::string &account_name() const + { + return m_account_name; + } + + const std::vector &account_key() const + { + return m_account_key; + } + + private: + std::string m_account_name; + std::vector m_account_key; + }; + + class shared_access_signature_credential : public storage_credential + { + public: + shared_access_signature_credential(const std::string &sas_token) + : m_sas_token(sas_token) + { + // If there is a question mark at the beginning of the sas token, erase it for easier processing in sign_request. + if (!m_sas_token.empty() && m_sas_token[0] == '?') + { + m_sas_token.erase(0, 1); + } + } + + AZURE_STORAGE_API void sign_request(const storage_request_base &r, http_base &h, const storage_url &url, const storage_headers &headers) const override; + AZURE_STORAGE_API std::string transform_url(std::string url) const override; + + private: + std::string m_sas_token; + }; + + class anonymous_credential : public storage_credential + { + public: + void sign_request(const storage_request_base &, http_base &, const storage_url &, const storage_headers &) const override {} + }; + +}} diff --git a/include/storage_errno.h b/include/storage_errno.h new file mode 100644 index 0000000..c8ba20e --- /dev/null +++ b/include/storage_errno.h @@ -0,0 +1,23 @@ +#pragma once +/* common errors*/ +const int invalid_parameters = 1200; +/* client level*/ +const int client_init_fail = 1300; +const int client_already_init = 1301; +const int client_not_init = 1302; +/* container level*/ +const int container_already_exists = 1400; +const int container_not_exists = 1401; +const int container_name_invalid = 1402; +const int container_create_fail = 1403; +const int container_delete_fail = 1404; +/* blob level*/ +const int blob__already_exists = 1500; +const int blob_not_exists = 1501; +const int blob_name_invalid = 1502; +const int blob_delete_fail = 1503; +const int blob_list_fail = 1504; +const int blob_copy_fail = 1505; +const int blob_no_content_range = 1506; +/* unknown error*/ +const int unknown_error = 1600; diff --git a/include/storage_outcome.h b/include/storage_outcome.h new file mode 100644 index 0000000..b5822ed --- /dev/null +++ b/include/storage_outcome.h @@ -0,0 +1,79 @@ +#pragma once + +#include + +#include "storage_EXPORTS.h" + +namespace azure { namespace storage_lite { + + class storage_error + { + public: + std::string code; + std::string code_name; + std::string message; + }; + + template + class storage_outcome + { + public: + storage_outcome() + : m_success(false) {} + + storage_outcome(RESPONSE_TYPE response) + : m_success(true), + m_response(std::move(response)) {} + + storage_outcome(storage_error error) + : m_success(false), + m_error(std::move(error)) {} + + bool success() const + { + return m_success; + } + + const storage_error &error() const + { + return m_error; + } + + const RESPONSE_TYPE &response() const + { + return m_response; + } + + private: + bool m_success; + storage_error m_error; + RESPONSE_TYPE m_response; + }; + + template<> + class storage_outcome + { + public: + storage_outcome() + : m_success(true) {} + + storage_outcome(storage_error error) + : m_success(false), + m_error(std::move(error)) {} + + bool success() const + { + return m_success; + } + + const storage_error &error() const + { + return m_error; + } + + private: + bool m_success; + storage_error m_error; + }; + +}} // azure::storage_lite diff --git a/include/storage_request_base.h b/include/storage_request_base.h new file mode 100644 index 0000000..4cb909f --- /dev/null +++ b/include/storage_request_base.h @@ -0,0 +1,36 @@ +#pragma once + +#include + +namespace azure { namespace storage_lite { + + class storage_account; + class http_base; + + class storage_request_base + { + public: + // TODO: create request ID for each request for future debugging purposes. + virtual std::string ms_client_request_id() const { return std::string(); } + + virtual void build_request(const storage_account &a, http_base &h) const = 0; + }; + + class blob_request_base : public storage_request_base + { + public: + virtual unsigned int timeout() const { return 0; } + virtual std::string if_modified_since() const { return std::string(); } + virtual std::string if_match() const { return std::string(); } + virtual std::string if_none_match() const { return std::string(); } + virtual std::string if_unmodified_since() const { return std::string(); } + virtual std::string ms_lease_id() const { return std::string(); } + }; + + class table_request_base : public storage_request_base {}; + + class queue_request_base : public storage_request_base {}; + + class file_request_base : public storage_request_base {}; + +}} diff --git a/include/storage_stream.h b/include/storage_stream.h new file mode 100644 index 0000000..01ceb59 --- /dev/null +++ b/include/storage_stream.h @@ -0,0 +1,149 @@ +#pragma once + +#include +#include +#include + +#include "storage_EXPORTS.h" + +namespace azure { namespace storage_lite { + + class storage_istream_helper + { + public: + storage_istream_helper(std::istream &stream) + : m_stream(stream) {} + + std::istream &istream() + { + return m_stream; + } + + private: + std::istream &m_stream; + }; + + class storage_istream + { + public: + storage_istream() {} + + storage_istream(std::istream &stream) + { + m_helper = std::make_shared(stream); + } + + storage_istream(std::shared_ptr stream) + { + m_stream = stream; + } + + void reset() + { + if (!valid()) + { + return; + } + istream().seekg(0); + } + + std::istream &istream() + { + if (m_helper) + { + return m_helper->istream(); + } + else { + return *m_stream; + } + } + + bool valid() const + { + return m_helper != nullptr || m_stream != nullptr; + } + + private: + std::shared_ptr m_helper; + std::shared_ptr m_stream; + }; + + class storage_ostream_helper + { + public: + storage_ostream_helper(std::ostream &stream) + : m_stream(stream) {} + + std::ostream &ostream() + { + return m_stream; + } + + private: + std::ostream &m_stream; + }; + + class storage_ostream + { + public: + storage_ostream() {} + + storage_ostream(std::ostream &stream) + { + m_initial = stream.tellp(); + m_helper = std::make_shared(stream); + } + + std::ostream &ostream() + { + return m_helper->ostream(); + } + + void reset() + { + if (!valid()) + { + return; + } + ostream().seekp(m_initial); + } + + bool valid() const + { + return m_helper != nullptr; + } + + private: + std::ostream::off_type m_initial; + std::shared_ptr m_helper; + }; + + class storage_iostream : public storage_istream, public storage_ostream + { + public: + static storage_iostream create_storage_stream() + { + return storage_iostream(std::make_shared()); + } + + static storage_iostream create_storage_stream(const std::string &str) + { + return storage_iostream(std::make_shared(str)); + } + + storage_iostream() {} + + storage_iostream(std::iostream &stream) + : storage_istream(stream), + storage_ostream(stream) {} + + private: + storage_iostream(std::shared_ptr s) + : storage_istream(*s), + storage_ostream(*s), + m_stream(s) {} + + std::shared_ptr m_stream; + }; + +}} // azure::storage_lite diff --git a/include/storage_url.h b/include/storage_url.h new file mode 100644 index 0000000..3cc1e3c --- /dev/null +++ b/include/storage_url.h @@ -0,0 +1,77 @@ +#pragma once + +#include +#include +#include + +#include "storage_EXPORTS.h" + +namespace azure { namespace storage_lite { + + std::string encode_url_path(const std::string& path); + + class storage_url + { + public: + storage_url & set_domain(const std::string &domain) + { + m_domain = domain; + return *this; + } + + const std::string &get_domain() const + { + return m_domain; + } + + storage_url &append_path(const std::string &segment) + { + m_path.append("/").append(segment); + return *this; + } + + const std::string &get_path() const + { + return m_path; + } + + std::string get_encoded_path() const + { + return encode_url_path(m_path); + } + + storage_url &add_query(const std::string &name, const std::string &value) + { + m_query[name].insert(value); + return *this; + } + + const std::map> &get_query() const + { + return m_query; + } + + AZURE_STORAGE_API std::string to_string() const; + + private: + std::string m_domain; + std::string m_path; + std::map> m_query; + }; + + class storage_headers + { + public: + std::string content_encoding; + std::string content_language; + std::string content_length; + std::string content_md5; + std::string content_type; + std::string if_modified_since; + std::string if_match; + std::string if_none_match; + std::string if_unmodified_since; + std::map ms_headers; + }; + +}} // azure::storage_lite diff --git a/include/tinyxml2.h b/include/tinyxml2.h new file mode 100644 index 0000000..a4769c8 --- /dev/null +++ b/include/tinyxml2.h @@ -0,0 +1,2130 @@ +/* +Original code by Lee Thomason (www.grinninglizard.com) + +This software is provided 'as-is', without any express or implied +warranty. In no event will the authors be held liable for any +damages arising from the use of this software. + +Permission is granted to anyone to use this software for any +purpose, including commercial applications, and to alter it and +redistribute it freely, subject to the following restrictions: + +1. The origin of this software must not be misrepresented; you must +not claim that you wrote the original software. If you use this +software in a product, an acknowledgment in the product documentation +would be appreciated but is not required. + +2. Altered source versions must be plainly marked as such, and +must not be misrepresented as being the original software. + +3. This notice may not be removed or altered from any source +distribution. +*/ + +#ifndef TINYXML2_INCLUDED +#define TINYXML2_INCLUDED + +#if defined(ANDROID_NDK) || defined(__BORLANDC__) || defined(__QNXNTO__) +# include +# include +# include +# include +# include +# include +#else +# include +# include +# include +# include +# include +# include +#endif + +/* + TODO: intern strings instead of allocation. +*/ +/* + gcc: + g++ -Wall -DDEBUG tinyxml2.cpp xmltest.cpp -o gccxmltest.exe + + Formatting, Artistic Style: + AStyle.exe --style=1tbs --indent-switches --break-closing-brackets --indent-preprocessor tinyxml2.cpp tinyxml2.h +*/ + +#if defined( _DEBUG ) || defined( DEBUG ) || defined (__DEBUG__) +# ifndef DEBUG +# define DEBUG +# endif +#endif + +#ifdef _MSC_VER +# pragma warning(push) +# pragma warning(disable: 4251) +#endif + +#ifdef _WIN32 +# ifdef TINYXML2_EXPORT +# define TINYXML2_LIB __declspec(dllexport) +# elif defined(TINYXML2_IMPORT) +# define TINYXML2_LIB __declspec(dllimport) +# else +# define TINYXML2_LIB +# endif +#else +# define TINYXML2_LIB +#endif + + +#if defined(DEBUG) +# if defined(_MSC_VER) +# // "(void)0," is for suppressing C4127 warning in "assert(false)", "assert(true)" and the like +# define TIXMLASSERT( x ) if ( !((void)0,(x))) { __debugbreak(); } //if ( !(x)) WinDebugBreak() +# elif defined (ANDROID_NDK) +# include +# define TIXMLASSERT( x ) if ( !(x)) { __android_log_assert( "assert", "grinliz", "ASSERT in '%s' at %d.", __FILE__, __LINE__ ); } +# else +# include +# define TIXMLASSERT assert +# endif +# else +# define TIXMLASSERT( x ) {} +#endif + + +#if defined(_MSC_VER) && (_MSC_VER >= 1400 ) && (!defined WINCE) +// Microsoft visual studio, version 2005 and higher. +/*int _snprintf_s( + char *buffer, + size_t sizeOfBuffer, + size_t count, + const char *format [, + argument] ... +);*/ +inline int TIXML_SNPRINTF( char* buffer, size_t size, const char* format, ... ) +{ + va_list va; + va_start( va, format ); + int result = vsnprintf_s( buffer, size, _TRUNCATE, format, va ); + va_end( va ); + return result; +} +#define TIXML_SSCANF sscanf_s +#elif defined WINCE +#define TIXML_SNPRINTF _snprintf +#define TIXML_SSCANF sscanf +#else +// GCC version 3 and higher +//#warning( "Using sn* functions." ) +#define TIXML_SNPRINTF snprintf +#define TIXML_SSCANF sscanf +#endif + +/* Versioning, past 1.0.14: + http://semver.org/ +*/ +static const int TIXML2_MAJOR_VERSION = 3; +static const int TIXML2_MINOR_VERSION = 0; +static const int TIXML2_PATCH_VERSION = 0; + +namespace tinyxml2 +{ +class XMLDocument; +class XMLElement; +class XMLAttribute; +class XMLComment; +class XMLText; +class XMLDeclaration; +class XMLUnknown; +class XMLPrinter; + +/* + A class that wraps strings. Normally stores the start and end + pointers into the XML file itself, and will apply normalization + and entity translation if actually read. Can also store (and memory + manage) a traditional char[] +*/ +class StrPair +{ +public: + enum { + NEEDS_ENTITY_PROCESSING = 0x01, + NEEDS_NEWLINE_NORMALIZATION = 0x02, + COLLAPSE_WHITESPACE = 0x04, + + TEXT_ELEMENT = NEEDS_ENTITY_PROCESSING | NEEDS_NEWLINE_NORMALIZATION, + TEXT_ELEMENT_LEAVE_ENTITIES = NEEDS_NEWLINE_NORMALIZATION, + ATTRIBUTE_NAME = 0, + ATTRIBUTE_VALUE = NEEDS_ENTITY_PROCESSING | NEEDS_NEWLINE_NORMALIZATION, + ATTRIBUTE_VALUE_LEAVE_ENTITIES = NEEDS_NEWLINE_NORMALIZATION, + COMMENT = NEEDS_NEWLINE_NORMALIZATION + }; + + StrPair() : _flags( 0 ), _start( 0 ), _end( 0 ) {} + ~StrPair(); + + void Set( char* start, char* end, int flags ) { + Reset(); + _start = start; + _end = end; + _flags = flags | NEEDS_FLUSH; + } + + const char* GetStr(); + + bool Empty() const { + return _start == _end; + } + + void SetInternedStr( const char* str ) { + Reset(); + _start = const_cast(str); + } + + void SetStr( const char* str, int flags=0 ); + + char* ParseText( char* in, const char* endTag, int strFlags ); + char* ParseName( char* in ); + + void TransferTo( StrPair* other ); + +private: + void Reset(); + void CollapseWhitespace(); + + enum { + NEEDS_FLUSH = 0x100, + NEEDS_DELETE = 0x200 + }; + + // After parsing, if *_end != 0, it can be set to zero. + int _flags; + char* _start; + char* _end; + + StrPair( const StrPair& other ); // not supported + void operator=( StrPair& other ); // not supported, use TransferTo() +}; + + +/* + A dynamic array of Plain Old Data. Doesn't support constructors, etc. + Has a small initial memory pool, so that low or no usage will not + cause a call to new/delete +*/ +template +class DynArray +{ +public: + DynArray() { + _mem = _pool; + _allocated = INIT; + _size = 0; + } + + ~DynArray() { + if ( _mem != _pool ) { + delete [] _mem; + } + } + + void Clear() { + _size = 0; + } + + void Push( T t ) { + TIXMLASSERT( _size < INT_MAX ); + EnsureCapacity( _size+1 ); + _mem[_size++] = t; + } + + T* PushArr( int count ) { + TIXMLASSERT( count >= 0 ); + TIXMLASSERT( _size <= INT_MAX - count ); + EnsureCapacity( _size+count ); + T* ret = &_mem[_size]; + _size += count; + return ret; + } + + T Pop() { + TIXMLASSERT( _size > 0 ); + return _mem[--_size]; + } + + void PopArr( int count ) { + TIXMLASSERT( _size >= count ); + _size -= count; + } + + bool Empty() const { + return _size == 0; + } + + T& operator[](int i) { + TIXMLASSERT( i>= 0 && i < _size ); + return _mem[i]; + } + + const T& operator[](int i) const { + TIXMLASSERT( i>= 0 && i < _size ); + return _mem[i]; + } + + const T& PeekTop() const { + TIXMLASSERT( _size > 0 ); + return _mem[ _size - 1]; + } + + int Size() const { + TIXMLASSERT( _size >= 0 ); + return _size; + } + + int Capacity() const { + return _allocated; + } + + const T* Mem() const { + return _mem; + } + + T* Mem() { + return _mem; + } + +private: + DynArray( const DynArray& ); // not supported + void operator=( const DynArray& ); // not supported + + void EnsureCapacity( int cap ) { + TIXMLASSERT( cap > 0 ); + if ( cap > _allocated ) { + TIXMLASSERT( cap <= INT_MAX / 2 ); + int newAllocated = cap * 2; + T* newMem = new T[newAllocated]; + memcpy( newMem, _mem, sizeof(T)*_size ); // warning: not using constructors, only works for PODs + if ( _mem != _pool ) { + delete [] _mem; + } + _mem = newMem; + _allocated = newAllocated; + } + } + + T* _mem; + T _pool[INIT]; + int _allocated; // objects allocated + int _size; // number objects in use +}; + + +/* + Parent virtual class of a pool for fast allocation + and deallocation of objects. +*/ +class MemPool +{ +public: + MemPool() {} + virtual ~MemPool() {} + + virtual int ItemSize() const = 0; + virtual void* Alloc() = 0; + virtual void Free( void* ) = 0; + virtual void SetTracked() = 0; + virtual void Clear() = 0; +}; + + +/* + Template child class to create pools of the correct type. +*/ +template< int SIZE > +class MemPoolT : public MemPool +{ +public: + MemPoolT() : _root(0), _currentAllocs(0), _nAllocs(0), _maxAllocs(0), _nUntracked(0) {} + ~MemPoolT() { + Clear(); + } + + void Clear() { + // Delete the blocks. + while( !_blockPtrs.Empty()) { + Block* b = _blockPtrs.Pop(); + delete b; + } + _root = 0; + _currentAllocs = 0; + _nAllocs = 0; + _maxAllocs = 0; + _nUntracked = 0; + } + + virtual int ItemSize() const { + return SIZE; + } + int CurrentAllocs() const { + return _currentAllocs; + } + + virtual void* Alloc() { + if ( !_root ) { + // Need a new block. + Block* block = new Block(); + _blockPtrs.Push( block ); + + for( int i=0; ichunk[i].next = &block->chunk[i+1]; + } + block->chunk[COUNT-1].next = 0; + _root = block->chunk; + } + void* result = _root; + _root = _root->next; + + ++_currentAllocs; + if ( _currentAllocs > _maxAllocs ) { + _maxAllocs = _currentAllocs; + } + _nAllocs++; + _nUntracked++; + return result; + } + + virtual void Free( void* mem ) { + if ( !mem ) { + return; + } + --_currentAllocs; + Chunk* chunk = static_cast( mem ); +#ifdef DEBUG + memset( chunk, 0xfe, sizeof(Chunk) ); +#endif + chunk->next = _root; + _root = chunk; + } + void Trace( const char* name ) { + printf( "Mempool %s watermark=%d [%dk] current=%d size=%d nAlloc=%d blocks=%d\n", + name, _maxAllocs, _maxAllocs*SIZE/1024, _currentAllocs, SIZE, _nAllocs, _blockPtrs.Size() ); + } + + void SetTracked() { + _nUntracked--; + } + + int Untracked() const { + return _nUntracked; + } + + // This number is perf sensitive. 4k seems like a good tradeoff on my machine. + // The test file is large, 170k. + // Release: VS2010 gcc(no opt) + // 1k: 4000 + // 2k: 4000 + // 4k: 3900 21000 + // 16k: 5200 + // 32k: 4300 + // 64k: 4000 21000 + enum { COUNT = (4*1024)/SIZE }; // Some compilers do not accept to use COUNT in private part if COUNT is private + +private: + MemPoolT( const MemPoolT& ); // not supported + void operator=( const MemPoolT& ); // not supported + + union Chunk { + Chunk* next; + char mem[SIZE]; + }; + struct Block { + Chunk chunk[COUNT]; + }; + DynArray< Block*, 10 > _blockPtrs; + Chunk* _root; + + int _currentAllocs; + int _nAllocs; + int _maxAllocs; + int _nUntracked; +}; + + + +/** + Implements the interface to the "Visitor pattern" (see the Accept() method.) + If you call the Accept() method, it requires being passed a XMLVisitor + class to handle callbacks. For nodes that contain other nodes (Document, Element) + you will get called with a VisitEnter/VisitExit pair. Nodes that are always leafs + are simply called with Visit(). + + If you return 'true' from a Visit method, recursive parsing will continue. If you return + false, no children of this node or its siblings will be visited. + + All flavors of Visit methods have a default implementation that returns 'true' (continue + visiting). You need to only override methods that are interesting to you. + + Generally Accept() is called on the XMLDocument, although all nodes support visiting. + + You should never change the document from a callback. + + @sa XMLNode::Accept() +*/ +class TINYXML2_LIB XMLVisitor +{ +public: + virtual ~XMLVisitor() {} + + /// Visit a document. + virtual bool VisitEnter( const XMLDocument& /*doc*/ ) { + return true; + } + /// Visit a document. + virtual bool VisitExit( const XMLDocument& /*doc*/ ) { + return true; + } + + /// Visit an element. + virtual bool VisitEnter( const XMLElement& /*element*/, const XMLAttribute* /*firstAttribute*/ ) { + return true; + } + /// Visit an element. + virtual bool VisitExit( const XMLElement& /*element*/ ) { + return true; + } + + /// Visit a declaration. + virtual bool Visit( const XMLDeclaration& /*declaration*/ ) { + return true; + } + /// Visit a text node. + virtual bool Visit( const XMLText& /*text*/ ) { + return true; + } + /// Visit a comment node. + virtual bool Visit( const XMLComment& /*comment*/ ) { + return true; + } + /// Visit an unknown node. + virtual bool Visit( const XMLUnknown& /*unknown*/ ) { + return true; + } +}; + +// WARNING: must match XMLDocument::_errorNames[] +enum XMLError { + XML_SUCCESS = 0, + XML_NO_ERROR = 0, + XML_NO_ATTRIBUTE, + XML_WRONG_ATTRIBUTE_TYPE, + XML_ERROR_FILE_NOT_FOUND, + XML_ERROR_FILE_COULD_NOT_BE_OPENED, + XML_ERROR_FILE_READ_ERROR, + XML_ERROR_ELEMENT_MISMATCH, + XML_ERROR_PARSING_ELEMENT, + XML_ERROR_PARSING_ATTRIBUTE, + XML_ERROR_IDENTIFYING_TAG, + XML_ERROR_PARSING_TEXT, + XML_ERROR_PARSING_CDATA, + XML_ERROR_PARSING_COMMENT, + XML_ERROR_PARSING_DECLARATION, + XML_ERROR_PARSING_UNKNOWN, + XML_ERROR_EMPTY_DOCUMENT, + XML_ERROR_MISMATCHED_ELEMENT, + XML_ERROR_PARSING, + XML_CAN_NOT_CONVERT_TEXT, + XML_NO_TEXT_NODE, + + XML_ERROR_COUNT +}; + + +/* + Utility functionality. +*/ +class XMLUtil +{ +public: + static const char* SkipWhiteSpace( const char* p ) { + TIXMLASSERT( p ); + while( IsWhiteSpace(*p) ) { + ++p; + } + TIXMLASSERT( p ); + return p; + } + static char* SkipWhiteSpace( char* p ) { + return const_cast( SkipWhiteSpace( const_cast(p) ) ); + } + + // Anything in the high order range of UTF-8 is assumed to not be whitespace. This isn't + // correct, but simple, and usually works. + static bool IsWhiteSpace( char p ) { + return !IsUTF8Continuation(p) && isspace( static_cast(p) ); + } + + inline static bool IsNameStartChar( unsigned char ch ) { + if ( ch >= 128 ) { + // This is a heuristic guess in attempt to not implement Unicode-aware isalpha() + return true; + } + if ( isalpha( ch ) ) { + return true; + } + return ch == ':' || ch == '_'; + } + + inline static bool IsNameChar( unsigned char ch ) { + return IsNameStartChar( ch ) + || isdigit( ch ) + || ch == '.' + || ch == '-'; + } + + inline static bool StringEqual( const char* p, const char* q, int nChar=INT_MAX ) { + if ( p == q ) { + return true; + } + int n = 0; + while( *p && *q && *p == *q && n(const_cast(this)->FirstChildElement( value )); + } + + /// Get the last child node, or null if none exists. + const XMLNode* LastChild() const { + return _lastChild; + } + + XMLNode* LastChild() { + return const_cast(const_cast(this)->LastChild() ); + } + + /** Get the last child element or optionally the last child + element with the specified name. + */ + const XMLElement* LastChildElement( const char* value=0 ) const; + + XMLElement* LastChildElement( const char* value=0 ) { + return const_cast(const_cast(this)->LastChildElement(value) ); + } + + /// Get the previous (left) sibling node of this node. + const XMLNode* PreviousSibling() const { + return _prev; + } + + XMLNode* PreviousSibling() { + return _prev; + } + + /// Get the previous (left) sibling element of this node, with an optionally supplied name. + const XMLElement* PreviousSiblingElement( const char* value=0 ) const ; + + XMLElement* PreviousSiblingElement( const char* value=0 ) { + return const_cast(const_cast(this)->PreviousSiblingElement( value ) ); + } + + /// Get the next (right) sibling node of this node. + const XMLNode* NextSibling() const { + return _next; + } + + XMLNode* NextSibling() { + return _next; + } + + /// Get the next (right) sibling element of this node, with an optionally supplied name. + const XMLElement* NextSiblingElement( const char* value=0 ) const; + + XMLElement* NextSiblingElement( const char* value=0 ) { + return const_cast(const_cast(this)->NextSiblingElement( value ) ); + } + + /** + Add a child node as the last (right) child. + If the child node is already part of the document, + it is moved from its old location to the new location. + Returns the addThis argument or 0 if the node does not + belong to the same document. + */ + XMLNode* InsertEndChild( XMLNode* addThis ); + + XMLNode* LinkEndChild( XMLNode* addThis ) { + return InsertEndChild( addThis ); + } + /** + Add a child node as the first (left) child. + If the child node is already part of the document, + it is moved from its old location to the new location. + Returns the addThis argument or 0 if the node does not + belong to the same document. + */ + XMLNode* InsertFirstChild( XMLNode* addThis ); + /** + Add a node after the specified child node. + If the child node is already part of the document, + it is moved from its old location to the new location. + Returns the addThis argument or 0 if the afterThis node + is not a child of this node, or if the node does not + belong to the same document. + */ + XMLNode* InsertAfterChild( XMLNode* afterThis, XMLNode* addThis ); + + /** + Delete all the children of this node. + */ + void DeleteChildren(); + + /** + Delete a child of this node. + */ + void DeleteChild( XMLNode* node ); + + /** + Make a copy of this node, but not its children. + You may pass in a Document pointer that will be + the owner of the new Node. If the 'document' is + null, then the node returned will be allocated + from the current Document. (this->GetDocument()) + + Note: if called on a XMLDocument, this will return null. + */ + virtual XMLNode* ShallowClone( XMLDocument* document ) const = 0; + + /** + Test if 2 nodes are the same, but don't test children. + The 2 nodes do not need to be in the same Document. + + Note: if called on a XMLDocument, this will return false. + */ + virtual bool ShallowEqual( const XMLNode* compare ) const = 0; + + /** Accept a hierarchical visit of the nodes in the TinyXML-2 DOM. Every node in the + XML tree will be conditionally visited and the host will be called back + via the XMLVisitor interface. + + This is essentially a SAX interface for TinyXML-2. (Note however it doesn't re-parse + the XML for the callbacks, so the performance of TinyXML-2 is unchanged by using this + interface versus any other.) + + The interface has been based on ideas from: + + - http://www.saxproject.org/ + - http://c2.com/cgi/wiki?HierarchicalVisitorPattern + + Which are both good references for "visiting". + + An example of using Accept(): + @verbatim + XMLPrinter printer; + tinyxmlDoc.Accept( &printer ); + const char* xmlcstr = printer.CStr(); + @endverbatim + */ + virtual bool Accept( XMLVisitor* visitor ) const = 0; + + // internal + virtual char* ParseDeep( char*, StrPair* ); + +protected: + XMLNode( XMLDocument* ); + virtual ~XMLNode(); + + XMLDocument* _document; + XMLNode* _parent; + mutable StrPair _value; + + XMLNode* _firstChild; + XMLNode* _lastChild; + + XMLNode* _prev; + XMLNode* _next; + +private: + MemPool* _memPool; + void Unlink( XMLNode* child ); + static void DeleteNode( XMLNode* node ); + void InsertChildPreamble( XMLNode* insertThis ) const; + + XMLNode( const XMLNode& ); // not supported + XMLNode& operator=( const XMLNode& ); // not supported +}; + + +/** XML text. + + Note that a text node can have child element nodes, for example: + @verbatim + This is bold + @endverbatim + + A text node can have 2 ways to output the next. "normal" output + and CDATA. It will default to the mode it was parsed from the XML file and + you generally want to leave it alone, but you can change the output mode with + SetCData() and query it with CData(). +*/ +class TINYXML2_LIB XMLText : public XMLNode +{ + friend class XMLBase; + friend class XMLDocument; +public: + virtual bool Accept( XMLVisitor* visitor ) const; + + virtual XMLText* ToText() { + return this; + } + virtual const XMLText* ToText() const { + return this; + } + + /// Declare whether this should be CDATA or standard text. + void SetCData( bool isCData ) { + _isCData = isCData; + } + /// Returns true if this is a CDATA text element. + bool CData() const { + return _isCData; + } + + char* ParseDeep( char*, StrPair* endTag ); + virtual XMLNode* ShallowClone( XMLDocument* document ) const; + virtual bool ShallowEqual( const XMLNode* compare ) const; + +protected: + XMLText( XMLDocument* doc ) : XMLNode( doc ), _isCData( false ) {} + virtual ~XMLText() {} + +private: + bool _isCData; + + XMLText( const XMLText& ); // not supported + XMLText& operator=( const XMLText& ); // not supported +}; + + +/** An XML Comment. */ +class TINYXML2_LIB XMLComment : public XMLNode +{ + friend class XMLDocument; +public: + virtual XMLComment* ToComment() { + return this; + } + virtual const XMLComment* ToComment() const { + return this; + } + + virtual bool Accept( XMLVisitor* visitor ) const; + + char* ParseDeep( char*, StrPair* endTag ); + virtual XMLNode* ShallowClone( XMLDocument* document ) const; + virtual bool ShallowEqual( const XMLNode* compare ) const; + +protected: + XMLComment( XMLDocument* doc ); + virtual ~XMLComment(); + +private: + XMLComment( const XMLComment& ); // not supported + XMLComment& operator=( const XMLComment& ); // not supported +}; + + +/** In correct XML the declaration is the first entry in the file. + @verbatim + + @endverbatim + + TinyXML-2 will happily read or write files without a declaration, + however. + + The text of the declaration isn't interpreted. It is parsed + and written as a string. +*/ +class TINYXML2_LIB XMLDeclaration : public XMLNode +{ + friend class XMLDocument; +public: + virtual XMLDeclaration* ToDeclaration() { + return this; + } + virtual const XMLDeclaration* ToDeclaration() const { + return this; + } + + virtual bool Accept( XMLVisitor* visitor ) const; + + char* ParseDeep( char*, StrPair* endTag ); + virtual XMLNode* ShallowClone( XMLDocument* document ) const; + virtual bool ShallowEqual( const XMLNode* compare ) const; + +protected: + XMLDeclaration( XMLDocument* doc ); + virtual ~XMLDeclaration(); + +private: + XMLDeclaration( const XMLDeclaration& ); // not supported + XMLDeclaration& operator=( const XMLDeclaration& ); // not supported +}; + + +/** Any tag that TinyXML-2 doesn't recognize is saved as an + unknown. It is a tag of text, but should not be modified. + It will be written back to the XML, unchanged, when the file + is saved. + + DTD tags get thrown into XMLUnknowns. +*/ +class TINYXML2_LIB XMLUnknown : public XMLNode +{ + friend class XMLDocument; +public: + virtual XMLUnknown* ToUnknown() { + return this; + } + virtual const XMLUnknown* ToUnknown() const { + return this; + } + + virtual bool Accept( XMLVisitor* visitor ) const; + + char* ParseDeep( char*, StrPair* endTag ); + virtual XMLNode* ShallowClone( XMLDocument* document ) const; + virtual bool ShallowEqual( const XMLNode* compare ) const; + +protected: + XMLUnknown( XMLDocument* doc ); + virtual ~XMLUnknown(); + +private: + XMLUnknown( const XMLUnknown& ); // not supported + XMLUnknown& operator=( const XMLUnknown& ); // not supported +}; + + + +/** An attribute is a name-value pair. Elements have an arbitrary + number of attributes, each with a unique name. + + @note The attributes are not XMLNodes. You may only query the + Next() attribute in a list. +*/ +class TINYXML2_LIB XMLAttribute +{ + friend class XMLElement; +public: + /// The name of the attribute. + const char* Name() const; + + /// The value of the attribute. + const char* Value() const; + + /// The next attribute in the list. + const XMLAttribute* Next() const { + return _next; + } + + /** IntValue interprets the attribute as an integer, and returns the value. + If the value isn't an integer, 0 will be returned. There is no error checking; + use QueryIntValue() if you need error checking. + */ + int IntValue() const { + int i=0; + QueryIntValue( &i ); + return i; + } + /// Query as an unsigned integer. See IntValue() + unsigned UnsignedValue() const { + unsigned i=0; + QueryUnsignedValue( &i ); + return i; + } + /// Query as a boolean. See IntValue() + bool BoolValue() const { + bool b=false; + QueryBoolValue( &b ); + return b; + } + /// Query as a double. See IntValue() + double DoubleValue() const { + double d=0; + QueryDoubleValue( &d ); + return d; + } + /// Query as a float. See IntValue() + float FloatValue() const { + float f=0; + QueryFloatValue( &f ); + return f; + } + + /** QueryIntValue interprets the attribute as an integer, and returns the value + in the provided parameter. The function will return XML_NO_ERROR on success, + and XML_WRONG_ATTRIBUTE_TYPE if the conversion is not successful. + */ + XMLError QueryIntValue( int* value ) const; + /// See QueryIntValue + XMLError QueryUnsignedValue( unsigned int* value ) const; + /// See QueryIntValue + XMLError QueryBoolValue( bool* value ) const; + /// See QueryIntValue + XMLError QueryDoubleValue( double* value ) const; + /// See QueryIntValue + XMLError QueryFloatValue( float* value ) const; + + /// Set the attribute to a string value. + void SetAttribute( const char* value ); + /// Set the attribute to value. + void SetAttribute( int value ); + /// Set the attribute to value. + void SetAttribute( unsigned value ); + /// Set the attribute to value. + void SetAttribute( bool value ); + /// Set the attribute to value. + void SetAttribute( double value ); + /// Set the attribute to value. + void SetAttribute( float value ); + +private: + enum { BUF_SIZE = 200 }; + + XMLAttribute() : _next( 0 ), _memPool( 0 ) {} + virtual ~XMLAttribute() {} + + XMLAttribute( const XMLAttribute& ); // not supported + void operator=( const XMLAttribute& ); // not supported + void SetName( const char* name ); + + char* ParseDeep( char* p, bool processEntities ); + + mutable StrPair _name; + mutable StrPair _value; + XMLAttribute* _next; + MemPool* _memPool; +}; + + +/** The element is a container class. It has a value, the element name, + and can contain other elements, text, comments, and unknowns. + Elements also contain an arbitrary number of attributes. +*/ +class TINYXML2_LIB XMLElement : public XMLNode +{ + friend class XMLBase; + friend class XMLDocument; +public: + /// Get the name of an element (which is the Value() of the node.) + const char* Name() const { + return Value(); + } + /// Set the name of the element. + void SetName( const char* str, bool staticMem=false ) { + SetValue( str, staticMem ); + } + + virtual XMLElement* ToElement() { + return this; + } + virtual const XMLElement* ToElement() const { + return this; + } + virtual bool Accept( XMLVisitor* visitor ) const; + + /** Given an attribute name, Attribute() returns the value + for the attribute of that name, or null if none + exists. For example: + + @verbatim + const char* value = ele->Attribute( "foo" ); + @endverbatim + + The 'value' parameter is normally null. However, if specified, + the attribute will only be returned if the 'name' and 'value' + match. This allow you to write code: + + @verbatim + if ( ele->Attribute( "foo", "bar" ) ) callFooIsBar(); + @endverbatim + + rather than: + @verbatim + if ( ele->Attribute( "foo" ) ) { + if ( strcmp( ele->Attribute( "foo" ), "bar" ) == 0 ) callFooIsBar(); + } + @endverbatim + */ + const char* Attribute( const char* name, const char* value=0 ) const; + + /** Given an attribute name, IntAttribute() returns the value + of the attribute interpreted as an integer. 0 will be + returned if there is an error. For a method with error + checking, see QueryIntAttribute() + */ + int IntAttribute( const char* name ) const { + int i=0; + QueryIntAttribute( name, &i ); + return i; + } + /// See IntAttribute() + unsigned UnsignedAttribute( const char* name ) const { + unsigned i=0; + QueryUnsignedAttribute( name, &i ); + return i; + } + /// See IntAttribute() + bool BoolAttribute( const char* name ) const { + bool b=false; + QueryBoolAttribute( name, &b ); + return b; + } + /// See IntAttribute() + double DoubleAttribute( const char* name ) const { + double d=0; + QueryDoubleAttribute( name, &d ); + return d; + } + /// See IntAttribute() + float FloatAttribute( const char* name ) const { + float f=0; + QueryFloatAttribute( name, &f ); + return f; + } + + /** Given an attribute name, QueryIntAttribute() returns + XML_NO_ERROR, XML_WRONG_ATTRIBUTE_TYPE if the conversion + can't be performed, or XML_NO_ATTRIBUTE if the attribute + doesn't exist. If successful, the result of the conversion + will be written to 'value'. If not successful, nothing will + be written to 'value'. This allows you to provide default + value: + + @verbatim + int value = 10; + QueryIntAttribute( "foo", &value ); // if "foo" isn't found, value will still be 10 + @endverbatim + */ + XMLError QueryIntAttribute( const char* name, int* value ) const { + const XMLAttribute* a = FindAttribute( name ); + if ( !a ) { + return XML_NO_ATTRIBUTE; + } + return a->QueryIntValue( value ); + } + /// See QueryIntAttribute() + XMLError QueryUnsignedAttribute( const char* name, unsigned int* value ) const { + const XMLAttribute* a = FindAttribute( name ); + if ( !a ) { + return XML_NO_ATTRIBUTE; + } + return a->QueryUnsignedValue( value ); + } + /// See QueryIntAttribute() + XMLError QueryBoolAttribute( const char* name, bool* value ) const { + const XMLAttribute* a = FindAttribute( name ); + if ( !a ) { + return XML_NO_ATTRIBUTE; + } + return a->QueryBoolValue( value ); + } + /// See QueryIntAttribute() + XMLError QueryDoubleAttribute( const char* name, double* value ) const { + const XMLAttribute* a = FindAttribute( name ); + if ( !a ) { + return XML_NO_ATTRIBUTE; + } + return a->QueryDoubleValue( value ); + } + /// See QueryIntAttribute() + XMLError QueryFloatAttribute( const char* name, float* value ) const { + const XMLAttribute* a = FindAttribute( name ); + if ( !a ) { + return XML_NO_ATTRIBUTE; + } + return a->QueryFloatValue( value ); + } + + + /** Given an attribute name, QueryAttribute() returns + XML_NO_ERROR, XML_WRONG_ATTRIBUTE_TYPE if the conversion + can't be performed, or XML_NO_ATTRIBUTE if the attribute + doesn't exist. It is overloaded for the primitive types, + and is a generally more convenient replacement of + QueryIntAttribute() and related functions. + + If successful, the result of the conversion + will be written to 'value'. If not successful, nothing will + be written to 'value'. This allows you to provide default + value: + + @verbatim + int value = 10; + QueryAttribute( "foo", &value ); // if "foo" isn't found, value will still be 10 + @endverbatim + */ + int QueryAttribute( const char* name, int* value ) const { + return QueryIntAttribute( name, value ); + } + + int QueryAttribute( const char* name, unsigned int* value ) const { + return QueryUnsignedAttribute( name, value ); + } + + int QueryAttribute( const char* name, bool* value ) const { + return QueryBoolAttribute( name, value ); + } + + int QueryAttribute( const char* name, double* value ) const { + return QueryDoubleAttribute( name, value ); + } + + int QueryAttribute( const char* name, float* value ) const { + return QueryFloatAttribute( name, value ); + } + + /// Sets the named attribute to value. + void SetAttribute( const char* name, const char* value ) { + XMLAttribute* a = FindOrCreateAttribute( name ); + a->SetAttribute( value ); + } + /// Sets the named attribute to value. + void SetAttribute( const char* name, int value ) { + XMLAttribute* a = FindOrCreateAttribute( name ); + a->SetAttribute( value ); + } + /// Sets the named attribute to value. + void SetAttribute( const char* name, unsigned value ) { + XMLAttribute* a = FindOrCreateAttribute( name ); + a->SetAttribute( value ); + } + /// Sets the named attribute to value. + void SetAttribute( const char* name, bool value ) { + XMLAttribute* a = FindOrCreateAttribute( name ); + a->SetAttribute( value ); + } + /// Sets the named attribute to value. + void SetAttribute( const char* name, double value ) { + XMLAttribute* a = FindOrCreateAttribute( name ); + a->SetAttribute( value ); + } + /// Sets the named attribute to value. + void SetAttribute( const char* name, float value ) { + XMLAttribute* a = FindOrCreateAttribute( name ); + a->SetAttribute( value ); + } + + /** + Delete an attribute. + */ + void DeleteAttribute( const char* name ); + + /// Return the first attribute in the list. + const XMLAttribute* FirstAttribute() const { + return _rootAttribute; + } + /// Query a specific attribute in the list. + const XMLAttribute* FindAttribute( const char* name ) const; + + /** Convenience function for easy access to the text inside an element. Although easy + and concise, GetText() is limited compared to getting the XMLText child + and accessing it directly. + + If the first child of 'this' is a XMLText, the GetText() + returns the character string of the Text node, else null is returned. + + This is a convenient method for getting the text of simple contained text: + @verbatim + This is text + const char* str = fooElement->GetText(); + @endverbatim + + 'str' will be a pointer to "This is text". + + Note that this function can be misleading. If the element foo was created from + this XML: + @verbatim + This is text + @endverbatim + + then the value of str would be null. The first child node isn't a text node, it is + another element. From this XML: + @verbatim + This is text + @endverbatim + GetText() will return "This is ". + */ + const char* GetText() const; + + /** Convenience function for easy access to the text inside an element. Although easy + and concise, SetText() is limited compared to creating an XMLText child + and mutating it directly. + + If the first child of 'this' is a XMLText, SetText() sets its value to + the given string, otherwise it will create a first child that is an XMLText. + + This is a convenient method for setting the text of simple contained text: + @verbatim + This is text + fooElement->SetText( "Hullaballoo!" ); + Hullaballoo! + @endverbatim + + Note that this function can be misleading. If the element foo was created from + this XML: + @verbatim + This is text + @endverbatim + + then it will not change "This is text", but rather prefix it with a text element: + @verbatim + Hullaballoo!This is text + @endverbatim + + For this XML: + @verbatim + + @endverbatim + SetText() will generate + @verbatim + Hullaballoo! + @endverbatim + */ + void SetText( const char* inText ); + /// Convenience method for setting text inside and element. See SetText() for important limitations. + void SetText( int value ); + /// Convenience method for setting text inside and element. See SetText() for important limitations. + void SetText( unsigned value ); + /// Convenience method for setting text inside and element. See SetText() for important limitations. + void SetText( bool value ); + /// Convenience method for setting text inside and element. See SetText() for important limitations. + void SetText( double value ); + /// Convenience method for setting text inside and element. See SetText() for important limitations. + void SetText( float value ); + + /** + Convenience method to query the value of a child text node. This is probably best + shown by example. Given you have a document is this form: + @verbatim + + 1 + 1.4 + + @endverbatim + + The QueryIntText() and similar functions provide a safe and easier way to get to the + "value" of x and y. + + @verbatim + int x = 0; + float y = 0; // types of x and y are contrived for example + const XMLElement* xElement = pointElement->FirstChildElement( "x" ); + const XMLElement* yElement = pointElement->FirstChildElement( "y" ); + xElement->QueryIntText( &x ); + yElement->QueryFloatText( &y ); + @endverbatim + + @returns XML_SUCCESS (0) on success, XML_CAN_NOT_CONVERT_TEXT if the text cannot be converted + to the requested type, and XML_NO_TEXT_NODE if there is no child text to query. + + */ + XMLError QueryIntText( int* ival ) const; + /// See QueryIntText() + XMLError QueryUnsignedText( unsigned* uval ) const; + /// See QueryIntText() + XMLError QueryBoolText( bool* bval ) const; + /// See QueryIntText() + XMLError QueryDoubleText( double* dval ) const; + /// See QueryIntText() + XMLError QueryFloatText( float* fval ) const; + + // internal: + enum { + OPEN, // + CLOSED, // + CLOSING // + }; + int ClosingType() const { + return _closingType; + } + char* ParseDeep( char* p, StrPair* endTag ); + virtual XMLNode* ShallowClone( XMLDocument* document ) const; + virtual bool ShallowEqual( const XMLNode* compare ) const; + +private: + XMLElement( XMLDocument* doc ); + virtual ~XMLElement(); + XMLElement( const XMLElement& ); // not supported + void operator=( const XMLElement& ); // not supported + + XMLAttribute* FindAttribute( const char* name ) { + return const_cast(const_cast(this)->FindAttribute( name )); + } + XMLAttribute* FindOrCreateAttribute( const char* name ); + //void LinkAttribute( XMLAttribute* attrib ); + char* ParseAttributes( char* p ); + static void DeleteAttribute( XMLAttribute* attribute ); + + enum { BUF_SIZE = 200 }; + int _closingType; + // The attribute list is ordered; there is no 'lastAttribute' + // because the list needs to be scanned for dupes before adding + // a new attribute. + XMLAttribute* _rootAttribute; +}; + + +enum Whitespace { + PRESERVE_WHITESPACE, + COLLAPSE_WHITESPACE +}; + + +/** A Document binds together all the functionality. + It can be saved, loaded, and printed to the screen. + All Nodes are connected and allocated to a Document. + If the Document is deleted, all its Nodes are also deleted. +*/ +class TINYXML2_LIB XMLDocument : public XMLNode +{ + friend class XMLElement; +public: + /// constructor + XMLDocument( bool processEntities = true, Whitespace = PRESERVE_WHITESPACE ); + ~XMLDocument(); + + virtual XMLDocument* ToDocument() { + return this; + } + virtual const XMLDocument* ToDocument() const { + return this; + } + + /** + Parse an XML file from a character string. + Returns XML_NO_ERROR (0) on success, or + an errorID. + + You may optionally pass in the 'nBytes', which is + the number of bytes which will be parsed. If not + specified, TinyXML-2 will assume 'xml' points to a + null terminated string. + */ + XMLError Parse( const char* xml, size_t nBytes=(size_t)(-1) ); + + /** + Load an XML file from disk. + Returns XML_NO_ERROR (0) on success, or + an errorID. + */ + XMLError LoadFile( const char* filename ); + + /** + Load an XML file from disk. You are responsible + for providing and closing the FILE*. + + NOTE: The file should be opened as binary ("rb") + not text in order for TinyXML-2 to correctly + do newline normalization. + + Returns XML_NO_ERROR (0) on success, or + an errorID. + */ + XMLError LoadFile( FILE* ); + + /** + Save the XML file to disk. + Returns XML_NO_ERROR (0) on success, or + an errorID. + */ + XMLError SaveFile( const char* filename, bool compact = false ); + + /** + Save the XML file to disk. You are responsible + for providing and closing the FILE*. + + Returns XML_NO_ERROR (0) on success, or + an errorID. + */ + XMLError SaveFile( FILE* fp, bool compact = false ); + + bool ProcessEntities() const { + return _processEntities; + } + Whitespace WhitespaceMode() const { + return _whitespace; + } + + /** + Returns true if this document has a leading Byte Order Mark of UTF8. + */ + bool HasBOM() const { + return _writeBOM; + } + /** Sets whether to write the BOM when writing the file. + */ + void SetBOM( bool useBOM ) { + _writeBOM = useBOM; + } + + /** Return the root element of DOM. Equivalent to FirstChildElement(). + To get the first node, use FirstChild(). + */ + XMLElement* RootElement() { + return FirstChildElement(); + } + const XMLElement* RootElement() const { + return FirstChildElement(); + } + + /** Print the Document. If the Printer is not provided, it will + print to stdout. If you provide Printer, this can print to a file: + @verbatim + XMLPrinter printer( fp ); + doc.Print( &printer ); + @endverbatim + + Or you can use a printer to print to memory: + @verbatim + XMLPrinter printer; + doc.Print( &printer ); + // printer.CStr() has a const char* to the XML + @endverbatim + */ + void Print( XMLPrinter* streamer=0 ) const; + virtual bool Accept( XMLVisitor* visitor ) const; + + /** + Create a new Element associated with + this Document. The memory for the Element + is managed by the Document. + */ + XMLElement* NewElement( const char* name ); + /** + Create a new Comment associated with + this Document. The memory for the Comment + is managed by the Document. + */ + XMLComment* NewComment( const char* comment ); + /** + Create a new Text associated with + this Document. The memory for the Text + is managed by the Document. + */ + XMLText* NewText( const char* text ); + /** + Create a new Declaration associated with + this Document. The memory for the object + is managed by the Document. + + If the 'text' param is null, the standard + declaration is used.: + @verbatim + + @endverbatim + */ + XMLDeclaration* NewDeclaration( const char* text=0 ); + /** + Create a new Unknown associated with + this Document. The memory for the object + is managed by the Document. + */ + XMLUnknown* NewUnknown( const char* text ); + + /** + Delete a node associated with this document. + It will be unlinked from the DOM. + */ + void DeleteNode( XMLNode* node ); + + void SetError( XMLError error, const char* str1, const char* str2 ); + + /// Return true if there was an error parsing the document. + bool Error() const { + return _errorID != XML_NO_ERROR; + } + /// Return the errorID. + XMLError ErrorID() const { + return _errorID; + } + const char* ErrorName() const; + + /// Return a possibly helpful diagnostic location or string. + const char* GetErrorStr1() const { + return _errorStr1; + } + /// Return a possibly helpful secondary diagnostic location or string. + const char* GetErrorStr2() const { + return _errorStr2; + } + /// If there is an error, print it to stdout. + void PrintError() const; + + /// Clear the document, resetting it to the initial state. + void Clear(); + + // internal + char* Identify( char* p, XMLNode** node ); + + virtual XMLNode* ShallowClone( XMLDocument* /*document*/ ) const { + return 0; + } + virtual bool ShallowEqual( const XMLNode* /*compare*/ ) const { + return false; + } + +private: + XMLDocument( const XMLDocument& ); // not supported + void operator=( const XMLDocument& ); // not supported + + bool _writeBOM; + bool _processEntities; + XMLError _errorID; + Whitespace _whitespace; + const char* _errorStr1; + const char* _errorStr2; + char* _charBuffer; + + MemPoolT< sizeof(XMLElement) > _elementPool; + MemPoolT< sizeof(XMLAttribute) > _attributePool; + MemPoolT< sizeof(XMLText) > _textPool; + MemPoolT< sizeof(XMLComment) > _commentPool; + + static const char* _errorNames[XML_ERROR_COUNT]; + + void Parse(); +}; + + +/** + A XMLHandle is a class that wraps a node pointer with null checks; this is + an incredibly useful thing. Note that XMLHandle is not part of the TinyXML-2 + DOM structure. It is a separate utility class. + + Take an example: + @verbatim + + + + + + + @endverbatim + + Assuming you want the value of "attributeB" in the 2nd "Child" element, it's very + easy to write a *lot* of code that looks like: + + @verbatim + XMLElement* root = document.FirstChildElement( "Document" ); + if ( root ) + { + XMLElement* element = root->FirstChildElement( "Element" ); + if ( element ) + { + XMLElement* child = element->FirstChildElement( "Child" ); + if ( child ) + { + XMLElement* child2 = child->NextSiblingElement( "Child" ); + if ( child2 ) + { + // Finally do something useful. + @endverbatim + + And that doesn't even cover "else" cases. XMLHandle addresses the verbosity + of such code. A XMLHandle checks for null pointers so it is perfectly safe + and correct to use: + + @verbatim + XMLHandle docHandle( &document ); + XMLElement* child2 = docHandle.FirstChildElement( "Document" ).FirstChildElement( "Element" ).FirstChildElement().NextSiblingElement(); + if ( child2 ) + { + // do something useful + @endverbatim + + Which is MUCH more concise and useful. + + It is also safe to copy handles - internally they are nothing more than node pointers. + @verbatim + XMLHandle handleCopy = handle; + @endverbatim + + See also XMLConstHandle, which is the same as XMLHandle, but operates on const objects. +*/ +class TINYXML2_LIB XMLHandle +{ +public: + /// Create a handle from any node (at any depth of the tree.) This can be a null pointer. + XMLHandle( XMLNode* node ) { + _node = node; + } + /// Create a handle from a node. + XMLHandle( XMLNode& node ) { + _node = &node; + } + /// Copy constructor + XMLHandle( const XMLHandle& ref ) { + _node = ref._node; + } + /// Assignment + XMLHandle& operator=( const XMLHandle& ref ) { + _node = ref._node; + return *this; + } + + /// Get the first child of this handle. + XMLHandle FirstChild() { + return XMLHandle( _node ? _node->FirstChild() : 0 ); + } + /// Get the first child element of this handle. + XMLHandle FirstChildElement( const char* value=0 ) { + return XMLHandle( _node ? _node->FirstChildElement( value ) : 0 ); + } + /// Get the last child of this handle. + XMLHandle LastChild() { + return XMLHandle( _node ? _node->LastChild() : 0 ); + } + /// Get the last child element of this handle. + XMLHandle LastChildElement( const char* _value=0 ) { + return XMLHandle( _node ? _node->LastChildElement( _value ) : 0 ); + } + /// Get the previous sibling of this handle. + XMLHandle PreviousSibling() { + return XMLHandle( _node ? _node->PreviousSibling() : 0 ); + } + /// Get the previous sibling element of this handle. + XMLHandle PreviousSiblingElement( const char* _value=0 ) { + return XMLHandle( _node ? _node->PreviousSiblingElement( _value ) : 0 ); + } + /// Get the next sibling of this handle. + XMLHandle NextSibling() { + return XMLHandle( _node ? _node->NextSibling() : 0 ); + } + /// Get the next sibling element of this handle. + XMLHandle NextSiblingElement( const char* _value=0 ) { + return XMLHandle( _node ? _node->NextSiblingElement( _value ) : 0 ); + } + + /// Safe cast to XMLNode. This can return null. + XMLNode* ToNode() { + return _node; + } + /// Safe cast to XMLElement. This can return null. + XMLElement* ToElement() { + return ( ( _node == 0 ) ? 0 : _node->ToElement() ); + } + /// Safe cast to XMLText. This can return null. + XMLText* ToText() { + return ( ( _node == 0 ) ? 0 : _node->ToText() ); + } + /// Safe cast to XMLUnknown. This can return null. + XMLUnknown* ToUnknown() { + return ( ( _node == 0 ) ? 0 : _node->ToUnknown() ); + } + /// Safe cast to XMLDeclaration. This can return null. + XMLDeclaration* ToDeclaration() { + return ( ( _node == 0 ) ? 0 : _node->ToDeclaration() ); + } + +private: + XMLNode* _node; +}; + + +/** + A variant of the XMLHandle class for working with const XMLNodes and Documents. It is the + same in all regards, except for the 'const' qualifiers. See XMLHandle for API. +*/ +class TINYXML2_LIB XMLConstHandle +{ +public: + XMLConstHandle( const XMLNode* node ) { + _node = node; + } + XMLConstHandle( const XMLNode& node ) { + _node = &node; + } + XMLConstHandle( const XMLConstHandle& ref ) { + _node = ref._node; + } + + XMLConstHandle& operator=( const XMLConstHandle& ref ) { + _node = ref._node; + return *this; + } + + const XMLConstHandle FirstChild() const { + return XMLConstHandle( _node ? _node->FirstChild() : 0 ); + } + const XMLConstHandle FirstChildElement( const char* value=0 ) const { + return XMLConstHandle( _node ? _node->FirstChildElement( value ) : 0 ); + } + const XMLConstHandle LastChild() const { + return XMLConstHandle( _node ? _node->LastChild() : 0 ); + } + const XMLConstHandle LastChildElement( const char* _value=0 ) const { + return XMLConstHandle( _node ? _node->LastChildElement( _value ) : 0 ); + } + const XMLConstHandle PreviousSibling() const { + return XMLConstHandle( _node ? _node->PreviousSibling() : 0 ); + } + const XMLConstHandle PreviousSiblingElement( const char* _value=0 ) const { + return XMLConstHandle( _node ? _node->PreviousSiblingElement( _value ) : 0 ); + } + const XMLConstHandle NextSibling() const { + return XMLConstHandle( _node ? _node->NextSibling() : 0 ); + } + const XMLConstHandle NextSiblingElement( const char* _value=0 ) const { + return XMLConstHandle( _node ? _node->NextSiblingElement( _value ) : 0 ); + } + + + const XMLNode* ToNode() const { + return _node; + } + const XMLElement* ToElement() const { + return ( ( _node == 0 ) ? 0 : _node->ToElement() ); + } + const XMLText* ToText() const { + return ( ( _node == 0 ) ? 0 : _node->ToText() ); + } + const XMLUnknown* ToUnknown() const { + return ( ( _node == 0 ) ? 0 : _node->ToUnknown() ); + } + const XMLDeclaration* ToDeclaration() const { + return ( ( _node == 0 ) ? 0 : _node->ToDeclaration() ); + } + +private: + const XMLNode* _node; +}; + + +/** + Printing functionality. The XMLPrinter gives you more + options than the XMLDocument::Print() method. + + It can: + -# Print to memory. + -# Print to a file you provide. + -# Print XML without a XMLDocument. + + Print to Memory + + @verbatim + XMLPrinter printer; + doc.Print( &printer ); + SomeFunction( printer.CStr() ); + @endverbatim + + Print to a File + + You provide the file pointer. + @verbatim + XMLPrinter printer( fp ); + doc.Print( &printer ); + @endverbatim + + Print without a XMLDocument + + When loading, an XML parser is very useful. However, sometimes + when saving, it just gets in the way. The code is often set up + for streaming, and constructing the DOM is just overhead. + + The Printer supports the streaming case. The following code + prints out a trivially simple XML file without ever creating + an XML document. + + @verbatim + XMLPrinter printer( fp ); + printer.OpenElement( "foo" ); + printer.PushAttribute( "foo", "bar" ); + printer.CloseElement(); + @endverbatim +*/ +class TINYXML2_LIB XMLPrinter : public XMLVisitor +{ +public: + /** Construct the printer. If the FILE* is specified, + this will print to the FILE. Else it will print + to memory, and the result is available in CStr(). + If 'compact' is set to true, then output is created + with only required whitespace and newlines. + */ + XMLPrinter( FILE* file=0, bool compact = false, int depth = 0 ); + virtual ~XMLPrinter() {} + + /** If streaming, write the BOM and declaration. */ + void PushHeader( bool writeBOM, bool writeDeclaration ); + /** If streaming, start writing an element. + The element must be closed with CloseElement() + */ + void OpenElement( const char* name, bool compactMode=false ); + /// If streaming, add an attribute to an open element. + void PushAttribute( const char* name, const char* value ); + void PushAttribute( const char* name, int value ); + void PushAttribute( const char* name, unsigned value ); + void PushAttribute( const char* name, bool value ); + void PushAttribute( const char* name, double value ); + /// If streaming, close the Element. + virtual void CloseElement( bool compactMode=false ); + + /// Add a text node. + void PushText( const char* text, bool cdata=false ); + /// Add a text node from an integer. + void PushText( int value ); + /// Add a text node from an unsigned. + void PushText( unsigned value ); + /// Add a text node from a bool. + void PushText( bool value ); + /// Add a text node from a float. + void PushText( float value ); + /// Add a text node from a double. + void PushText( double value ); + + /// Add a comment + void PushComment( const char* comment ); + + void PushDeclaration( const char* value ); + void PushUnknown( const char* value ); + + virtual bool VisitEnter( const XMLDocument& /*doc*/ ); + virtual bool VisitExit( const XMLDocument& /*doc*/ ) { + return true; + } + + virtual bool VisitEnter( const XMLElement& element, const XMLAttribute* attribute ); + virtual bool VisitExit( const XMLElement& element ); + + virtual bool Visit( const XMLText& text ); + virtual bool Visit( const XMLComment& comment ); + virtual bool Visit( const XMLDeclaration& declaration ); + virtual bool Visit( const XMLUnknown& unknown ); + + /** + If in print to memory mode, return a pointer to + the XML file in memory. + */ + const char* CStr() const { + return _buffer.Mem(); + } + /** + If in print to memory mode, return the size + of the XML file in memory. (Note the size returned + includes the terminating null.) + */ + int CStrSize() const { + return _buffer.Size(); + } + /** + If in print to memory mode, reset the buffer to the + beginning. + */ + void ClearBuffer() { + _buffer.Clear(); + _buffer.Push(0); + } + +protected: + virtual bool CompactMode( const XMLElement& ) { return _compactMode; } + + /** Prints out the space before an element. You may override to change + the space and tabs used. A PrintSpace() override should call Print(). + */ + virtual void PrintSpace( int depth ); + void Print( const char* format, ... ); + + void SealElementIfJustOpened(); + bool _elementJustOpened; + DynArray< const char*, 10 > _stack; + +private: + void PrintString( const char*, bool restrictedEntitySet ); // prints out, after detecting entities. + + bool _firstElement; + FILE* _fp; + int _depth; + int _textDepth; + bool _processEntities; + bool _compactMode; + + enum { + ENTITY_RANGE = 64, + BUF_SIZE = 200 + }; + bool _entityFlag[ENTITY_RANGE]; + bool _restrictedEntityFlag[ENTITY_RANGE]; + + DynArray< char, 20 > _buffer; +}; + + +} // tinyxml2 + +#if defined(_MSC_VER) +# pragma warning(pop) +#endif + +#endif // TINYXML2_INCLUDED diff --git a/include/tinyxml2_parser.h b/include/tinyxml2_parser.h new file mode 100644 index 0000000..af8f4c9 --- /dev/null +++ b/include/tinyxml2_parser.h @@ -0,0 +1,45 @@ +#pragma once + +#include "tinyxml2.h" + +#include "storage_EXPORTS.h" + +#include "storage_outcome.h" +#include "xml_parser_base.h" + +namespace azure { namespace storage_lite { + + class tinyxml2_parser : public xml_parser_base + { + public: + AZURE_STORAGE_API storage_error parse_storage_error(const std::string &xml) const override; + + AZURE_STORAGE_API list_constainers_segmented_response parse_list_constainers_segmented_response(const std::string &xml) const override; + + AZURE_STORAGE_API list_blobs_response parse_list_blobs_response(const std::string &xml) const override; + + AZURE_STORAGE_API list_blobs_segmented_response parse_list_blobs_segmented_response(const std::string &xml) const override; + + AZURE_STORAGE_API get_block_list_response parse_get_block_list_response(const std::string &xml) const override; + + AZURE_STORAGE_API get_page_ranges_response parse_get_page_ranges_response(const std::string &xml) const override; + + private: + AZURE_STORAGE_API std::string parse_text(tinyxml2::XMLElement *ele, const std::string &name) const; + + AZURE_STORAGE_API unsigned long long parse_long(tinyxml2::XMLElement *ele, const std::string &name) const; + + AZURE_STORAGE_API list_containers_item parse_list_containers_item(tinyxml2::XMLElement *ele) const; + + AZURE_STORAGE_API list_blobs_item parse_list_blobs_item(tinyxml2::XMLElement *ele) const; + + AZURE_STORAGE_API std::vector> parse_blob_metadata(tinyxml2::XMLElement *ele) const; + + AZURE_STORAGE_API list_blobs_segmented_item parse_list_blobs_segmented_item(tinyxml2::XMLElement *ele, bool is_directory) const; + + AZURE_STORAGE_API get_block_list_item parse_get_block_list_item(tinyxml2::XMLElement *ele) const; + + AZURE_STORAGE_API get_page_ranges_item parse_get_page_ranges_item(tinyxml2::XMLElement *ele) const; + }; + +}} // azure::storage_lite diff --git a/include/todo/get_blob_metadata_request.h b/include/todo/get_blob_metadata_request.h new file mode 100644 index 0000000..82aa2c8 --- /dev/null +++ b/include/todo/get_blob_metadata_request.h @@ -0,0 +1,78 @@ +#pragma once + +#include +#include +#include + +#include "storage_request.h" +#include "process_storage_request.h" +#include "sign_storage_request.h" +#include "storage_url.h" +#include "utility.h" +#include "constants.h" +#include "common.h" + +namespace azure { + namespace storage_lite { + namespace experimental { + + struct get_blob_metadata_request_base : blob_request_base { + std::string date() { return std::string(); } + std::string range() { return std::string(); } + std::string content_encoding() { return std::string(); } + std::string content_language() { return std::string(); } + unsigned long long content_length() { return 0; } + std::string content_md5() { return std::string(); } + std::string content_type() { return std::string(); } + std::string if_modified_since() { return std::string(); } + std::string if_match() { return std::string(); } + std::string if_none_match() { return std::string(); } + std::string if_unmodified_since() { return std::string(); } + }; + + template + struct storage_request_processor>::value>> { + static inline void process(Request &&r, Http &&h) { + //static_assert(std::is_same::value, "get_blob_request::use_https != Request::use_https"); + std::string method = constant::http_head; + h.set_method(http_method::head); + + storage_url url; + url.use_https = r.use_https(); + url.use_custom_endpoint = r.use_custom_endpoint(); + url.domain = r.use_custom_endpoint() ? r.custom_endpoint() : r.account() + r.endpoint_suffix(); + url.append_path(r.container()).append_path(r.blob()); + url.add_optional_query(constant::query_comp, constant::query_comp_metadata); + url.add_optional_query(constant::query_snapshot, r.snapshot()); + url.add_optional_query(constant::query_timeout, (r.timeout() != std::numeric_limits::max() ? std::to_string(r.timeout()) : std::string())); + h.set_url(url.to_string()); + + // TODO: move the lines below to set blob property request base when implementing it. + // details::add_optional_header(h, constant::header_content_encoding, r.content_encoding()); + // details::add_optional_header(h, constant::header_content_language, r.content_language()); + // details::add_optional_header(h, constant::header_content_length, (r.content_length() != 0 ? std::to_string(r.content_length()) : std::string())); + // details::add_optional_header(h, constant::header_content_md5, r.content_md5()); + // details::add_optional_header(h, constant::header_content_type, r.content_type()); + // details::add_optional_header(h, constant::header_date, r.date()); + // details::add_optional_header(h, constant::header_if_modified_since, r.if_modified_since()); + // details::add_optional_header(h, constant::header_if_match, r.if_match()); + // details::add_optional_header(h, constant::header_if_none_match, r.if_none_match()); + // details::add_optional_header(h, constant::header_if_unmodified_since, r.if_unmodified_since()); + // details::add_optional_header(h, constant::header_range, r.range()); + + std::map ms_headers; + if (!r.lease_id().empty()) { + details::add_ms_header(h, ms_headers, constant::header_ms_lease_id, r.lease_id()); + } + if (!r.client_request_id().empty()) { + details::add_ms_header(h, ms_headers, constant::header_ms_client_request_id, r.client_request_id()); + } + details::add_ms_header(h, ms_headers, constant::header_ms_date, details::get_date()); + details::add_ms_header(h, ms_headers, constant::header_ms_version, constant::header_value_storage_version); + + h.add_header(constant::header_authorization, sign_storage_request(r, method, url, ms_headers)); + } + }; + } + } +} diff --git a/include/todo/get_blob_properties_request.h b/include/todo/get_blob_properties_request.h new file mode 100644 index 0000000..cd73847 --- /dev/null +++ b/include/todo/get_blob_properties_request.h @@ -0,0 +1,76 @@ +#pragma once + +#include +#include +#include + +#include "storage_request.h" +#include "process_storage_request.h" +#include "sign_storage_request.h" +#include "storage_url.h" +#include "utility.h" +#include "constants.h" +#include "common.h" + +namespace azure { + namespace storage_lite { + namespace experimental { + + struct get_blob_properties_request : blob_request_base { + std::string date() { return std::string(); } + std::string range() { return std::string(); } + std::string content_encoding() { return std::string(); } + std::string content_language() { return std::string(); } + unsigned long long content_length() { return 0; } + std::string content_md5() { return std::string(); } + std::string content_type() { return std::string(); } + std::string if_modified_since() { return std::string(); } + std::string if_match() { return std::string(); } + std::string if_none_match() { return std::string(); } + std::string if_unmodified_since() { return std::string(); } + }; + + template + struct storage_request_processor>::value>> { + static inline void process(Request &&r, Http &&h) { + //static_assert(std::is_same::value, "get_blob_request::use_https != Request::use_https"); + std::string method = constant::http_head; + h.set_method(http_method::head); + + storage_url url; + url.use_https = r.use_https(); + url.use_custom_endpoint = r.use_custom_endpoint(); + url.domain = r.use_custom_endpoint() ? r.custom_endpoint() : r.account() + r.endpoint_suffix(); + url.append_path(r.container()).append_path(r.blob()); + url.add_optional_query(constant::query_snapshot, r.snapshot()); + url.add_optional_query(constant::query_timeout, (r.timeout() != std::numeric_limits::max() ? std::to_string(r.timeout()) : std::string())); + h.set_url(url.to_string()); + + details::add_optional_header(h, constant::header_content_encoding, r.content_encoding()); + details::add_optional_header(h, constant::header_content_language, r.content_language()); + details::add_optional_header(h, constant::header_content_length, (r.content_length() != 0 ? std::to_string(r.content_length()) : std::string())); + details::add_optional_header(h, constant::header_content_md5, r.content_md5()); + details::add_optional_header(h, constant::header_content_type, r.content_type()); + details::add_optional_header(h, constant::header_date, r.date()); + details::add_optional_header(h, constant::header_if_modified_since, r.if_modified_since()); + details::add_optional_header(h, constant::header_if_match, r.if_match()); + details::add_optional_header(h, constant::header_if_none_match, r.if_none_match()); + details::add_optional_header(h, constant::header_if_unmodified_since, r.if_unmodified_since()); + details::add_optional_header(h, constant::header_range, r.range()); + + std::map ms_headers; + if (!r.lease_id().empty()) { + details::add_ms_header(h, ms_headers, constant::header_ms_lease_id, r.lease_id()); + } + if (!r.client_request_id().empty()) { + details::add_ms_header(h, ms_headers, constant::header_ms_client_request_id, r.client_request_id()); + } + details::add_ms_header(h, ms_headers, constant::header_ms_date, details::get_date()); + details::add_ms_header(h, ms_headers, constant::header_ms_version, constant::header_value_storage_version); + + h.add_header(constant::header_authorization, sign_storage_request(r, method, url, ms_headers)); + } + }; + } + } +} diff --git a/include/todo/query_entities_request.h b/include/todo/query_entities_request.h new file mode 100644 index 0000000..ef49f56 --- /dev/null +++ b/include/todo/query_entities_request.h @@ -0,0 +1,87 @@ +#pragma once + +#include +#include +#include + +#include "defaults.h" +#include "process_storage_request.h" +#include "sign_storage_request.h" +#include "storage_url.h" +#include "utility.h" +#include "constants.h" + +namespace azure { + namespace storage_lite { + namespace experimental { + + struct query_entities_request_base : table_request_base { + std::string account() = delete; + std::vector key() = delete; + std::string table() = delete; + + std::string sas_token() { return defaults::sas_token; }; + std::string custom_endpoint() { return defaults::custom_endpoint; } + bool https() { return defaults::https; } + std::string endpoint_suffix() { return defaults::endpoint_suffix; } + + std::string partition_key() { return defaults::partition_key; } + std::string row_key() { return defaults::row_key; } + + azure::storage::experimental::payload_format payload_format() { return azure::storage::experimental::payload_format::json_nometadata; } + std::string ms_client_request_id() { return defaults::ms_client_request_id; } + }; + + template + struct storage_request_processor::type>::value>::type> : storage_request_processor_base { + static inline void process(Request &&r, Http &&h) { + auto method = http_method::get; + h.set_method(method); + + storage_url url{ get_domain(r, constants::table_prefix) }; + std::string table = r.table(); + if (r.partition_key() != defaults::partition_key && r.row_key() != defaults::row_key) { + table.append("(PartitionKey='").append(r.partition_key()).append("',RowKey='").append(r.row_key()).append("')"); + } + else { + table.append("()"); + } + url.append_path(table); + + std::string transform_url = url.to_string(); + if (r.sas_token() != defaults::sas_token) { + transform_url.append("?").append(constants::query_api_version).append("=").append(constants::header_value_storage_version); + transform_url.append("&").append(r.sas_token()); + } + h.set_url(transform_url); + + h.add_header(constants::header_user_agent, defaults::user_agent); + + std::map ms_headers; + if (r.ms_client_request_id() != defaults::ms_client_request_id) { + h.add_header(constants::header_ms_client_request_id, r.ms_client_request_id()); + ms_headers[constants::header_ms_client_request_id] = r.ms_client_request_id(); + } + + auto ms_date = get_ms_date(date_format::rfc_1123); + h.add_header(constants::header_ms_date, ms_date); + ms_headers[constants::header_ms_date] = std::move(ms_date); + + h.add_header(constants::header_ms_version, constants::header_value_storage_version); + ms_headers[constants::header_ms_version] = constants::header_value_storage_version; + + if (r.payload_format() == payload_format::json_nometadata) { + h.add_header(constants::header_accept, constants::header_payload_format_nometadata); + } + else { + h.add_header(constants::header_accept, constants::header_payload_format_fullmetadata); + } + + if (r.sas_token() == defaults::sas_token) { + ;//h.add_header(constants::header_authorization, sign_storage_request(r, get_method(method), std::move(url), std::move(headers_to_sign), std::move(ms_headers), std::move(sh))); + } + } + }; + } + } +} diff --git a/include/todo/set_blob_metadata_request.h b/include/todo/set_blob_metadata_request.h new file mode 100644 index 0000000..13736dc --- /dev/null +++ b/include/todo/set_blob_metadata_request.h @@ -0,0 +1,82 @@ +#pragma once + +#include +#include +#include + +#include "storage_request.h" +#include "process_storage_request.h" +#include "sign_storage_request.h" +#include "storage_url.h" +#include "utility.h" +#include "constants.h" +#include "common.h" + +namespace azure { + namespace storage_lite { + namespace experimental { + + struct set_blob_metadata_request : blob_request { + std::string date() { return std::string(); } + std::string range() { return std::string(); } + std::string content_encoding() { return std::string(); } + std::string content_language() { return std::string(); } + unsigned long long content_length() { return 0; } + std::string content_md5() { return std::string(); } + std::string content_type() { return std::string(); } + std::string if_modified_since() { return std::string(); } + std::string if_match() { return std::string(); } + std::string if_none_match() { return std::string(); } + std::string if_unmodified_since() { return std::string(); } + std::map meta() { return std::map(); } + }; + + template + struct storage_request_processor>::value>> { + static inline void process(Request &&r, Http &&h) { + //static_assert(std::is_same::value, "get_blob_request::use_https != Request::use_https"); + std::string method = constant::http_put; + h.set_method(http_method::put); + + storage_url url; + url.use_https = r.use_https(); + url.use_custom_endpoint = r.use_custom_endpoint(); + url.domain = r.use_custom_endpoint() ? r.custom_endpoint() : r.account() + r.endpoint_suffix(); + url.append_path(r.container()).append_path(r.blob()); + url.add_optional_query(constant::query_comp, constant::query_comp_metadata); + url.add_optional_query(constant::query_timeout, (r.timeout() != std::numeric_limits::max() ? std::to_string(r.timeout()) : std::string())); + h.set_url(url.to_string()); + + details::add_optional_header(h, constant::header_content_encoding, r.content_encoding()); + details::add_optional_header(h, constant::header_content_language, r.content_language()); + details::add_optional_header(h, constant::header_content_length, (r.content_length() != 0 ? std::to_string(r.content_length()) : std::string())); + details::add_optional_header(h, constant::header_content_md5, r.content_md5()); + details::add_optional_header(h, constant::header_content_type, r.content_type()); + details::add_optional_header(h, constant::header_date, r.date()); + details::add_optional_header(h, constant::header_if_modified_since, r.if_modified_since()); + details::add_optional_header(h, constant::header_if_match, r.if_match()); + details::add_optional_header(h, constant::header_if_none_match, r.if_none_match()); + details::add_optional_header(h, constant::header_if_unmodified_since, r.if_unmodified_since()); + details::add_optional_header(h, constant::header_range, r.range()); + + std::map ms_headers; + if (!r.lease_id().empty()) { + details::add_ms_header(h, ms_headers, constant::header_ms_lease_id, r.lease_id()); + } + if (!r.client_request_id().empty()) { + details::add_ms_header(h, ms_headers, constant::header_ms_client_request_id, r.client_request_id()); + } + details::add_ms_header(h, ms_headers, constant::header_ms_date, details::get_date()); + details::add_ms_header(h, ms_headers, constant::header_ms_version, constant::header_value_storage_version); + + auto m = r.meta(); + for (auto it = m.begin(); it != m.end(); it++) { + details::add_ms_header(h, ms_headers, constant::header_ms_meta_prefix + it->first, it->second); + } + + h.add_header(constant::header_authorization, sign_storage_request(r, method, url, ms_headers)); + } + }; + } + } +} diff --git a/include/utility.h b/include/utility.h new file mode 100644 index 0000000..19c1486 --- /dev/null +++ b/include/utility.h @@ -0,0 +1,184 @@ +#pragma once + +#include +#include + +#include "storage_EXPORTS.h" + +#include "common.h" +#include "constants.h" +#include "http_base.h" +#include "storage_request_base.h" +#include "storage_url.h" + +namespace azure { namespace storage_lite { + + enum class date_format + { + rfc_1123, + iso_8601 + }; + + AZURE_STORAGE_API std::string get_ms_date(date_format format); + + AZURE_STORAGE_API std::string get_ms_range(unsigned long long start_byte, unsigned long long end_byte); + + AZURE_STORAGE_API std::string get_http_verb(http_base::http_method method); + + inline void add_optional_query(storage_url &url, const std::string &name, unsigned int value) + { + if (value > 0) + { + url.add_query(name, std::to_string(value)); + } + } + + inline void add_optional_query(storage_url &url, const std::string &name, const std::string &value) + { + if (!value.empty()) + { + url.add_query(name, value); + } + } + + AZURE_STORAGE_API void add_access_condition_headers(http_base &h, storage_headers &headers, const blob_request_base &r); + + inline void add_optional_header(http_base &h, const std::string &name, const std::string &value) + { + if (!value.empty()) + { + h.add_header(name, value); + } + } + + inline void add_optional_content_encoding(http_base &h, storage_headers &headers, const std::string &value) + { + if (!value.empty()) + { + h.add_header(constants::header_content_encoding, value); + headers.content_encoding = value; + } + } + + inline void add_optional_content_language(http_base &h, storage_headers &headers, const std::string &value) + { + if (!value.empty()) + { + h.add_header(constants::header_content_language, value); + headers.content_language = value; + } + } + + inline void add_content_length(http_base &h, storage_headers &headers, unsigned int length) + { + std::string value = std::to_string(length); + h.add_header(constants::header_content_length, value); + if (length > 0) + { + headers.content_length = value; + } + } + + inline void add_optional_content_md5(http_base &h, storage_headers &headers, const std::string &value) + { + if (!value.empty()) + { + h.add_header(constants::header_content_md5, value); + headers.content_md5 = value; + } + } + + inline void add_optional_content_type(http_base &h, storage_headers &headers, const std::string &value) + { + if (!value.empty()) + { + h.add_header(constants::header_content_type, value); + headers.content_type = value; + } + } + + inline void add_ms_header(http_base &h, storage_headers &headers, const std::string &name, const std::string &value, bool optional = false) + { + if (!optional || !value.empty()) + { + h.add_header(name, value); + headers.ms_headers[name] = value; + } + } + + inline void add_ms_header(http_base &h, storage_headers &headers, const std::string &name, unsigned long long value, bool optional = false) + { + if (!optional || (value != std::numeric_limits::max())) + { + h.add_header(name, std::to_string(value)); + headers.ms_headers[name] = std::to_string(value); + } + } + + inline void add_metadata_header(http_base &h, storage_headers &headers, const std::string &name, const std::string &value, bool optional = false) + { + add_ms_header(h, headers, constants::header_ms_meta_prefix + name, value, optional); + } + + AZURE_STORAGE_API bool retryable(http_base::http_code status_code); + + AZURE_STORAGE_API std::string get_uuid(); + + + inline bool unsuccessful(http_base::http_code status_code) + { + return !(status_code >= 200 && status_code < 300); + } + + inline lease_status parse_lease_status(const std::string &value) + { + if (value == "locked") + { + return lease_status::locked; + } + else if (value == "unlocked") + { + return lease_status::unlocked; + } + return lease_status::unlocked; + } + + inline lease_state parse_lease_state(const std::string &value) + { + if (value == "available") + { + return lease_state::available; + } + else if (value == "leased") + { + return lease_state::leased; + } + else if (value == "expired") + { + return lease_state::expired; + } + else if (value == "breaking") + { + return lease_state::breaking; + } + else if (value == "broken") + { + return lease_state::broken; + } + return lease_state::available; + } + + inline lease_duration parse_lease_duration(const std::string &value) + { + if (value == "infinite") + { + return lease_duration::infinite; + } + else if (value == "fixed") + { + return lease_duration::fixed; + } + return lease_duration::none; + } + +}} // azure::storage_lite diff --git a/include/xml_parser_base.h b/include/xml_parser_base.h new file mode 100644 index 0000000..db1b0b5 --- /dev/null +++ b/include/xml_parser_base.h @@ -0,0 +1,63 @@ +#pragma once + +#include +#include + +#include "common.h" +#include "list_containers_request_base.h" +#include "list_blobs_request_base.h" +#include "get_block_list_request_base.h" +#include "get_page_ranges_request_base.h" + +namespace azure { namespace storage_lite { + + class xml_parser_base + { + public: + virtual storage_error parse_storage_error(const std::string &) const = 0; + + template + RESPONSE_TYPE parse_response(const std::string &) const {} + + virtual list_constainers_segmented_response parse_list_constainers_segmented_response(const std::string &xml) const = 0; + + virtual list_blobs_response parse_list_blobs_response(const std::string &xml) const = 0; + + virtual list_blobs_segmented_response parse_list_blobs_segmented_response(const std::string &xml) const = 0; + + virtual get_block_list_response parse_get_block_list_response(const std::string &xml) const = 0; + + virtual get_page_ranges_response parse_get_page_ranges_response(const std::string &xml) const = 0; + }; + + template<> + inline list_constainers_segmented_response xml_parser_base::parse_response(const std::string &xml) const + { + return parse_list_constainers_segmented_response(xml); + } + + template<> + inline list_blobs_response xml_parser_base::parse_response(const std::string &xml) const + { + return parse_list_blobs_response(xml); + } + + template<> + inline list_blobs_segmented_response xml_parser_base::parse_response(const std::string &xml) const + { + return parse_list_blobs_segmented_response(xml); + } + + template<> + inline get_block_list_response xml_parser_base::parse_response(const std::string &xml) const + { + return parse_get_block_list_response(xml); + } + + template<> + inline get_page_ranges_response xml_parser_base::parse_response(const std::string &xml) const + { + return parse_get_page_ranges_response(xml); + } + +}} // azure::storage_lite diff --git a/include/xml_writer.h b/include/xml_writer.h new file mode 100644 index 0000000..9acd2f5 --- /dev/null +++ b/include/xml_writer.h @@ -0,0 +1,56 @@ +#pragma once + +#include +#include + +#include "common.h" +#include "put_block_list_request_base.h" + +namespace azure { namespace storage_lite { + + class xml_writer + { + public: + static std::string write_block_list(const std::vector &items) + { + std::string xml; + xml.append(""); + xml.append(""); + + for (const auto &b : items) + { + switch (b.type) + { + case put_block_list_request_base::block_type::committed: + xml.append(""); + break; + case put_block_list_request_base::block_type::uncommitted: + xml.append(""); + break; + case put_block_list_request_base::block_type::latest: + xml.append(""); + break; + } + + xml.append(b.id); + + switch (b.type) + { + case put_block_list_request_base::block_type::committed: + xml.append(""); + break; + case put_block_list_request_base::block_type::uncommitted: + xml.append(""); + break; + case put_block_list_request_base::block_type::latest: + xml.append(""); + break; + } + } + + xml.append(""); + return xml; + } + }; + +}} diff --git a/sample/CMakeLists.txt b/sample/CMakeLists.txt new file mode 100644 index 0000000..5364945 --- /dev/null +++ b/sample/CMakeLists.txt @@ -0,0 +1,4 @@ +cmake_minimum_required(VERSION 2.8) + +add_executable(azure-storage-sample sample.cpp) +target_link_libraries(azure-storage-sample azure-storage-lite) \ No newline at end of file diff --git a/sample/Makefile b/sample/Makefile new file mode 100644 index 0000000..81a001d --- /dev/null +++ b/sample/Makefile @@ -0,0 +1,3 @@ + +all: + g++ --std=c++11 sample.cpp -o run -I../include -L.. -lazure-storage -lcurl diff --git a/sample/sample.cpp b/sample/sample.cpp new file mode 100644 index 0000000..c5b6753 --- /dev/null +++ b/sample/sample.cpp @@ -0,0 +1,106 @@ + +#include +#include +#include +#include + +#include "storage_credential.h" +#include "storage_account.h" +#include "blob/blob_client.h" + + +using namespace azure::storage_lite; + +void checkstatus() +{ + if(errno == 0) + { + printf("Success\n"); + } + else + { + printf("Fail\n"); + } +} + +int main() +{ + std::string account_name = "YOUR_ACCOUNT_NAME"; + std::string account_key = "YOUR_ACCOUNT_KEY"; + std::shared_ptr cred = std::make_shared(account_name, account_key); + std::shared_ptr account = std::make_shared(account_name, cred, false); + auto bC = std::make_shared(account, 10); + //auto f1 = bc.list_containers(""); + //f1.wait(); + // + std::string containerName = "jasontest1"; + std::string blobName = "test.txt"; + std::string destContainerName = "jasontest1"; + std::string destBlobName = "test.txt.copy"; + std::string uploadFileName = "test.txt"; + std::string downloadFileName = "download.txt"; + + bool exists = true; + blob_client_wrapper bc(bC); + + exists = bc.container_exists(containerName); + + if(!exists) + { + bc.create_container(containerName); + assert(errno == 0); + } + + assert(errno == 0); + exists = bc.blob_exists(containerName, "testsss.txt"); + assert(errno == 0); + assert(!exists); + std::cout <<"Start upload Blob: " << blobName << std::endl; + bc.upload_file_to_blob(uploadFileName, containerName, blobName); + std::cout <<"Error upload Blob: " << errno << std::endl; + assert(errno == 0); + + exists = bc.blob_exists(containerName, blobName); + assert(errno == 0); + assert(exists); + + auto blobProperty = bc.get_blob_property(containerName, blobName); + assert(errno == 0); + std::cout <<"Size of BLob: " << blobProperty.size << std::endl; + + auto blobs = bc.list_blobs_segmented(containerName, "/", "", ""); + std::cout <<"Size of BLobs: " << blobs.blobs.size() << std::endl; + std::cout <<"Error Size of BLobs: " << errno << std::endl; + assert(errno == 0); + + time_t last_modified; + bc.download_blob_to_file(containerName, blobName, downloadFileName, last_modified); + std::cout <<"Download Blob done: " << errno << std::endl; + assert(errno == 0); + + exists = bc.container_exists(destContainerName); + + if(!exists) + { + bc.create_container(destContainerName); + assert(errno == 0); + } + + // copy blob + bc.start_copy(containerName, blobName, destContainerName, destBlobName); + auto property = bc.get_blob_property(destContainerName, destBlobName); + std::cout << "Copy status: " << property.copy_status <sign_request(r, h, url, headers); + } + +}} // azure::storage_lite diff --git a/src/base64.cpp b/src/base64.cpp new file mode 100644 index 0000000..f0775e5 --- /dev/null +++ b/src/base64.cpp @@ -0,0 +1,233 @@ +#include +#include + +#include "base64.h" + +namespace { + bool is_ascii(char c) + { + return static_cast(c) >= 0; + } +} + +namespace azure { namespace storage_lite { + + static const char* _base64_enctbl = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + + const std::array _base64_dectbl = + { { 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 62, 255, 255, 255, 63, + 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 255, 255, 255, 254, 255, 255, + 255, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, + 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 255, 255, 255, 255, 255, + 255, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, + 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 255, 255, 255, 255, 255 } }; + + struct _triple_byte + { + unsigned char _1_1 : 2; + unsigned char _0 : 6; + unsigned char _2_1 : 4; + unsigned char _1_2 : 4; + unsigned char _3 : 6; + unsigned char _2_2 : 2; + }; + + std::string to_base64(const std::vector &input) + { + auto size = input.size(); + auto ptr = input.data(); + + std::string result; + result.reserve((size / 3 + 1) * 4); + + for (; size >= 3;) + { + const _triple_byte* record = reinterpret_cast(ptr); + unsigned char idx0 = record->_0; + unsigned char idx1 = (record->_1_1 << 4) | record->_1_2; + unsigned char idx2 = (record->_2_1 << 2) | record->_2_2; + unsigned char idx3 = record->_3; + result.push_back(char(_base64_enctbl[idx0])); + result.push_back(char(_base64_enctbl[idx1])); + result.push_back(char(_base64_enctbl[idx2])); + result.push_back(char(_base64_enctbl[idx3])); + size -= 3; + ptr += 3; + } + + switch (size) + { + case 1: { + const _triple_byte* record = reinterpret_cast(ptr); + unsigned char idx0 = record->_0; + unsigned char idx1 = (record->_1_1 << 4); + result.push_back(char(_base64_enctbl[idx0])); + result.push_back(char(_base64_enctbl[idx1])); + result.push_back('='); + result.push_back('='); + break; + } + case 2: + { + const _triple_byte* record = reinterpret_cast(ptr); + unsigned char idx0 = record->_0; + unsigned char idx1 = (record->_1_1 << 4) | record->_1_2; + unsigned char idx2 = (record->_2_1 << 2); + result.push_back(char(_base64_enctbl[idx0])); + result.push_back(char(_base64_enctbl[idx1])); + result.push_back(char(_base64_enctbl[idx2])); + result.push_back('='); + break; + } + } + + return result; + } + + std::vector from_base64(const std::string &input) + { + std::vector result; + + if (input.empty()) + return result; + + size_t padding = 0; + + // Validation + { + auto size = input.size(); + + if ((size % 4) != 0) + { + throw std::runtime_error("length of base64 string is not an even multiple of 4"); + } + + for (auto iter = input.begin(); iter != input.end(); ++iter, --size) + { + const auto ch = *iter; + if (!is_ascii(ch)) + { + throw std::runtime_error("invalid character found in base64 string"); + } + const size_t ch_sz = static_cast(ch); + if (ch_sz >= _base64_dectbl.size() || _base64_dectbl[ch_sz] == 255) + { + throw std::runtime_error("invalid character found in base64 string"); + } + if (_base64_dectbl[ch_sz] == 254) + { + padding++; + // padding only at the end + if (size > 2) + { + throw std::runtime_error("invalid padding character found in base64 string"); + } + if (size == 2) + { + const auto ch2 = *(iter + 1); + if (!is_ascii(ch2)) + { + throw std::runtime_error("invalid padding character found in base64 string"); + } + const size_t ch2_sz = static_cast(ch2); + if (ch2_sz >= _base64_dectbl.size() || _base64_dectbl[ch2_sz] != 254) + { + throw std::runtime_error("invalid padding character found in base64 string"); + } + } + } + } + } + + + auto size = input.size(); + const char* ptr = &input[0]; + + auto outsz = (size / 4) * 3; + outsz -= padding; + + result.resize(outsz); + + size_t idx = 0; + for (; size > 4; ++idx) + { + unsigned char target[3]; + std::memset(target, 0, sizeof(target)); + _triple_byte* record = reinterpret_cast<_triple_byte*>(target); + + unsigned char val0 = _base64_dectbl[ptr[0]]; + unsigned char val1 = _base64_dectbl[ptr[1]]; + unsigned char val2 = _base64_dectbl[ptr[2]]; + unsigned char val3 = _base64_dectbl[ptr[3]]; + + record->_0 = val0; + record->_1_1 = val1 >> 4; + result[idx] = target[0]; + + record->_1_2 = val1 & 0xF; + record->_2_1 = val2 >> 2; + result[++idx] = target[1]; + + record->_2_2 = val2 & 0x3; + record->_3 = val3 & 0x3F; + result[++idx] = target[2]; + + ptr += 4; + size -= 4; + } + + // Handle the last four bytes separately, to avoid having the conditional statements + // in all the iterations (a performance issue). + { + unsigned char target[3]; + std::memset(target, 0, sizeof(target)); + _triple_byte* record = reinterpret_cast<_triple_byte*>(target); + + unsigned char val0 = _base64_dectbl[ptr[0]]; + unsigned char val1 = _base64_dectbl[ptr[1]]; + unsigned char val2 = _base64_dectbl[ptr[2]]; + unsigned char val3 = _base64_dectbl[ptr[3]]; + + record->_0 = val0; + record->_1_1 = val1 >> 4; + result[idx] = target[0]; + + record->_1_2 = val1 & 0xF; + if (val2 != 254) + { + record->_2_1 = val2 >> 2; + result[++idx] = target[1]; + } + else + { + // There shouldn't be any information (ones) in the unused bits, + if (record->_1_2 != 0) + { + throw std::runtime_error("Invalid end of base64 string"); + } + return result; + } + + record->_2_2 = val2 & 0x3; + if (val3 != 254) + { + record->_3 = val3 & 0x3F; + result[++idx] = target[2]; + } + else + { + // There shouldn't be any information (ones) in the unused bits. + if (record->_2_2 != 0) + { + throw std::runtime_error("Invalid end of base64 string"); + } + return result; + } + } + + return result; + } + +}} // azure::storage_lite diff --git a/src/blob/blob_client.cpp b/src/blob/blob_client.cpp new file mode 100644 index 0000000..be2ada4 --- /dev/null +++ b/src/blob/blob_client.cpp @@ -0,0 +1,384 @@ +#include +#include +#include "blob/blob_client.h" + +#include "blob/download_blob_request.h" +#include "blob/create_block_blob_request.h" +#include "blob/delete_blob_request.h" +#include "blob/copy_blob_request.h" +#include "blob/create_container_request.h" +#include "blob/delete_container_request.h" +#include "blob/list_containers_request.h" +#include "blob/list_blobs_request.h" +#include "blob/get_block_list_request.h" +#include "blob/get_blob_property_request.h" +#include "blob/get_container_property_request.h" +#include "blob/put_block_request.h" +#include "blob/put_block_list_request.h" +#include "blob/append_block_request.h" +#include "blob/put_page_request.h" +#include "blob/get_page_ranges_request.h" + +#include "executor.h" +#include "utility.h" +#include "tinyxml2_parser.h" +#include + +namespace azure { namespace storage_lite { + +namespace { + +// Return content size from content-range header or -1 if cannot be obtained. +ssize_t get_length_from_content_range(const std::string &header) +{ + const auto pos = header.rfind('/'); + if (std::string::npos == pos) { + return -1; + } + const auto lengthStr = header.substr(pos + 1); + ssize_t result; + if (!(std::istringstream(lengthStr) >> result)) { + return -1; + } + return result; +} + +} // noname namespace + +storage_outcome blob_client::get_chunk_to_stream_sync(const std::string &container, const std::string &blob, unsigned long long offset, unsigned long long size, std::ostream &os) +{ + auto http = m_client->get_handle(); + auto request = std::make_shared(container, blob); + if (size > 0) { + request->set_start_byte(offset); + request->set_end_byte(offset + size - 1); + } + else { + request->set_start_byte(offset); + } + + http->set_output_stream(storage_ostream(os)); + + // TODO: async submit transfered to sync operation. This can be utilized. + const auto response = async_executor::submit(m_account, request, http, m_context).get(); + if (response.success()) + { + chunk_property property{}; + property.etag = http->get_header(constants::header_etag); + property.totalSize = get_length_from_content_range(http->get_header(constants::header_content_range)); + std::istringstream(http->get_header(constants::header_content_length)) >> property.size; + property.last_modified = curl_getdate(http->get_header(constants::header_last_modified).c_str(), NULL); + return storage_outcome(property); + } + return storage_outcome(storage_error(response.error())); +} + +std::future> blob_client::download_blob_to_stream(const std::string &container, const std::string &blob, unsigned long long offset, unsigned long long size, std::ostream &os) +{ + auto http = m_client->get_handle(); + + auto request = std::make_shared(container, blob); + + if (size > 0) { + request->set_start_byte(offset); + request->set_end_byte(offset + size - 1); + } + else { + request->set_start_byte(offset); + } + + http->set_output_stream(storage_ostream(os)); + + return async_executor::submit(m_account, request, http, m_context); +} + +std::future> blob_client::upload_block_blob_from_stream(const std::string &container, const std::string &blob, std::istream &is, const std::vector> &metadata) +{ + auto http = m_client->get_handle(); + + auto request = std::make_shared(container, blob); + + auto cur = is.tellg(); + is.seekg(0, std::ios_base::end); + auto end = is.tellg(); + is.seekg(cur); + request->set_content_length(static_cast(end - cur)); + if (metadata.size() > 0) + { + request->set_metadata(metadata); + } + + http->set_input_stream(storage_istream(is)); + + return async_executor::submit(m_account, request, http, m_context); +} + +std::future> blob_client::delete_blob(const std::string &container, const std::string &blob, bool delete_snapshots) +{ + auto http = m_client->get_handle(); + + auto request = std::make_shared(container, blob, delete_snapshots); + + return async_executor::submit(m_account, request, http, m_context); +} + +std::future> blob_client::create_container(const std::string &container) +{ + auto http = m_client->get_handle(); + + auto request = std::make_shared(container); + + return async_executor::submit(m_account, request, http, m_context); +} + +std::future> blob_client::delete_container(const std::string &container) +{ + auto http = m_client->get_handle(); + + auto request = std::make_shared(container); + + return async_executor::submit(m_account, request, http, m_context); +} + +storage_outcome blob_client::get_container_property(const std::string &container) +{ + auto http = m_client->get_handle(); + + auto request = std::make_shared(container); + + // TODO: async submit transfered to sync operation. This can be utilized. + auto response = async_executor::submit(m_account, request, http, m_context).get(); + container_property containerProperty(true); + if (response.success()) + { + containerProperty.etag = http->get_header(constants::header_etag); + + auto& headers = http->get_headers(); + for (auto iter = headers.begin(); iter != headers.end(); ++iter) + { + if (iter->first.find("x-ms-metadata-") == 0) + { + containerProperty.metadata.push_back(std::make_pair(iter->first, iter->second)); + } + } + } + else + { + containerProperty.set_valid(false); + } + return storage_outcome(containerProperty); +} + +std::future> blob_client::list_containers_segmented(const std::string &prefix, const std::string& continuation_token, const int max_result, bool include_metadata) +{ + auto http = m_client->get_handle(); + + auto request = std::make_shared(prefix, include_metadata); + request->set_maxresults(max_result); + request->set_marker(continuation_token); + + return async_executor::submit(m_account, request, http, m_context); +} + +std::future> blob_client::list_blobs_segmented(const std::string &container, const std::string &delimiter, const std::string &continuation_token, const std::string &prefix, int max_results) +{ + auto http = m_client->get_handle(); + + auto request = std::make_shared(container, delimiter, continuation_token, prefix); + request->set_maxresults(max_results); + request->set_includes(list_blobs_request_base::include::metadata); + + return async_executor::submit(m_account, request, http, m_context); +} + +std::future> blob_client::get_block_list(const std::string &container, const std::string &blob) +{ + auto http = m_client->get_handle(); + + auto request = std::make_shared(container, blob); + + return async_executor::submit(m_account, request, http, m_context); +} + +storage_outcome blob_client::get_blob_property(const std::string &container, const std::string &blob) +{ + auto http = m_client->get_handle(); + + auto request = std::make_shared(container, blob); + + auto response = async_executor::submit(m_account, request, http, m_context).get(); + blob_property blobProperty(true); + if (response.success()) + { + blobProperty.cache_control = http->get_header(constants::header_cache_control); + blobProperty.content_disposition = http->get_header(constants::header_content_disposition); + blobProperty.content_encoding = http->get_header(constants::header_content_encoding); + blobProperty.content_language = http->get_header(constants::header_content_language); + blobProperty.content_md5 = http->get_header(constants::header_content_md5); + blobProperty.content_type = http->get_header(constants::header_content_type); + blobProperty.etag = http->get_header(constants::header_etag); + blobProperty.copy_status = http->get_header(constants::header_ms_copy_status); + blobProperty.last_modified = curl_getdate(http->get_header(constants::header_last_modified).c_str(), NULL); + std::string::size_type sz = 0; + std::string contentLength = http->get_header(constants::header_content_length); + if(contentLength.length() > 0) + { + blobProperty.size = std::stoull(contentLength, &sz, 0); + } + + auto& headers = http->get_headers(); + for (auto iter = headers.begin(); iter != headers.end(); ++iter) + { + if (iter->first.find("x-ms-meta-") == 0) + { + // We need to strip ten characters from the front of the key to account for "x-ms-meta-", and two characters from the end of the value, to account for the "\r\n". + blobProperty.metadata.push_back(std::make_pair(iter->first.substr(10), iter->second.substr(0, iter->second.size() - 2))); + } + } + } + else + { + blobProperty.set_valid(false); + } + return storage_outcome(blobProperty); +} + +std::future> blob_client::upload_block_from_stream(const std::string &container, const std::string &blob, const std::string &blockid, std::istream &is) +{ + auto http = m_client->get_handle(); + + auto request = std::make_shared(container, blob, blockid); + + auto cur = is.tellg(); + is.seekg(0, std::ios_base::end); + auto end = is.tellg(); + is.seekg(cur); + request->set_content_length(static_cast(end - cur)); + + http->set_input_stream(storage_istream(is)); + + return async_executor::submit(m_account, request, http, m_context); +} + +std::future> blob_client::put_block_list(const std::string &container, const std::string &blob, const std::vector &block_list, const std::vector> &metadata) +{ + auto http = m_client->get_handle(); + + auto request = std::make_shared(container, blob); + request->set_block_list(block_list); + if (metadata.size() > 0) + { + request->set_metadata(metadata); + } + + return async_executor::submit(m_account, request, http, m_context); +} + +std::future> blob_client::create_append_blob(const std::string &container, const std::string &blob) +{ + auto http = m_client->get_handle(); + + auto request = std::make_shared(container, blob); + + return async_executor::submit(m_account, request, http, m_context); +} + +std::future> blob_client::append_block_from_stream(const std::string &container, const std::string &blob, std::istream &is) +{ + auto http = m_client->get_handle(); + + auto request = std::make_shared(container, blob); + + auto cur = is.tellg(); + is.seekg(0, std::ios_base::end); + auto end = is.tellg(); + is.seekg(cur); + request->set_content_length(static_cast(end - cur)); + + http->set_input_stream(storage_istream(is)); + + return async_executor::submit(m_account, request, http, m_context); +} + +std::future> blob_client::create_page_blob(const std::string &container, const std::string &blob, unsigned long long size) +{ + auto http = m_client->get_handle(); + + auto request = std::make_shared(container, blob, size); + + return async_executor::submit(m_account, request, http, m_context); +} + +std::future> blob_client::put_page_from_stream(const std::string &container, const std::string &blob, unsigned long long offset, unsigned long long size, std::istream &is) +{ + auto http = m_client->get_handle(); + + auto request = std::make_shared(container, blob); + if (size > 0) + { + request->set_start_byte(offset); + request->set_end_byte(offset + size - 1); + } + else + { + request->set_start_byte(offset); + } + + auto cur = is.tellg(); + is.seekg(0, std::ios_base::end); + auto end = is.tellg(); + is.seekg(cur); + auto stream_size = static_cast(end - cur); + request->set_content_length(stream_size); + + http->set_input_stream(storage_istream(is)); + + return async_executor::submit(m_account, request, http, m_context); +} + +std::future> blob_client::clear_page(const std::string &container, const std::string &blob, unsigned long long offset, unsigned long long size) +{ + auto http = m_client->get_handle(); + + auto request = std::make_shared(container, blob, true); + if (size > 0) + { + request->set_start_byte(offset); + request->set_end_byte(offset + size - 1); + } + else + { + request->set_start_byte(offset); + } + + return async_executor::submit(m_account, request, http, m_context); +} + +std::future> blob_client::get_page_ranges(const std::string &container, const std::string &blob, unsigned long long offset, unsigned long long size) +{ + auto http = m_client->get_handle(); + + auto request = std::make_shared(container, blob); + if (size > 0) + { + request->set_start_byte(offset); + request->set_end_byte(offset + size - 1); + } + else + { + request->set_start_byte(offset); + } + + return async_executor::submit(m_account, request, http, m_context); +} + +std::future> blob_client::start_copy(const std::string &sourceContainer, const std::string &sourceBlob, const std::string &destContainer, const std::string &destBlob) +{ + auto http = m_client->get_handle(); + + auto request = std::make_shared(sourceContainer, sourceBlob, destContainer, destBlob); + + return async_executor::submit(m_account, request, http, m_context); +} + +}} diff --git a/src/blob/blob_client_wrapper.cpp b/src/blob/blob_client_wrapper.cpp new file mode 100644 index 0000000..c98aefa --- /dev/null +++ b/src/blob/blob_client_wrapper.cpp @@ -0,0 +1,882 @@ +/* C++ interface wrapper for blob client + * No exceptions will throw. + */ +#include +#ifdef __linux__ +#include +#else +#include +#endif //__linux__ +#include +#include +#include +#include + +#include "blob/blob_client.h" +#include "storage_errno.h" + +namespace azure { namespace storage_lite { + + const unsigned long long DOWNLOAD_CHUNK_SIZE = 16 * 1024 * 1024; + const long long MIN_UPLOAD_CHUNK_SIZE = 16 * 1024 * 1024; + const long long MAX_BLOB_SIZE = 5242880000000; // 4.77TB + + class mempool + { + public: + ~mempool() + { + while(!m_buffers.empty()) + { + auto buffer = m_buffers.front(); + delete[] buffer; + m_buffers.pop(); + } + } + + char* get_buffer() + { + std::lock_guard lg(m_buffers_mutex); + if(m_buffers.empty()) + { + char* buffer = new char[s_block_size]; + return buffer; + } + else + { + char* buffer = m_buffers.front(); + m_buffers.pop(); + return buffer; + } + } + void release_buffer(char *buffer) + { + std::lock_guard lg(m_buffers_mutex); + m_buffers.push(buffer); + } + private: + std::queue m_buffers; + std::mutex m_buffers_mutex; + static const size_t s_block_size = 4*1024*1024; + }; + static mempool mpool; + off_t get_file_size(const char* path); + + static const char* _base64_enctbl = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + std::string to_base64(const char* base, size_t length) + { + std::string result; + for(int offset = 0; length - offset >= 3; offset += 3) + { + const char* ptr = base + offset; + unsigned char idx0 = ptr[0] >> 2; + unsigned char idx1 = ((ptr[0]&0x3)<<4)| ptr[1] >> 4; + unsigned char idx2 = ((ptr[1]&0xF)<<2)| ptr[2] >> 6; + unsigned char idx3 = ptr[2]&0x3F; + result.push_back(_base64_enctbl[idx0]); + result.push_back(_base64_enctbl[idx1]); + result.push_back(_base64_enctbl[idx2]); + result.push_back(_base64_enctbl[idx3]); + } + switch(length % 3) + { + case 1: + { + + const char* ptr = base + length - 1; + unsigned char idx0 = ptr[0] >> 2; + unsigned char idx1 = ((ptr[0]&0x3)<<4); + result.push_back(_base64_enctbl[idx0]); + result.push_back(_base64_enctbl[idx1]); + result.push_back('='); + result.push_back('='); + break; + } + case 2: + { + + const char* ptr = base + length - 2; + unsigned char idx0 = ptr[0] >> 2; + unsigned char idx1 = ((ptr[0]&0x3)<<4)| ptr[1] >> 4; + unsigned char idx2 = ((ptr[1]&0xF)<<2); + result.push_back(_base64_enctbl[idx0]); + result.push_back(_base64_enctbl[idx1]); + result.push_back(_base64_enctbl[idx2]); + result.push_back('='); + break; + } + } + return result; + } + + blob_client_wrapper blob_client_wrapper::blob_client_wrapper_init(const std::string &account_name, const std::string &account_key, const std::string &sas_token, const unsigned int concurrency) + { + return blob_client_wrapper_init(account_name, account_key, sas_token, concurrency, false, NULL); + } + + + blob_client_wrapper blob_client_wrapper::blob_client_wrapper_init(const std::string &account_name, const std::string &account_key, const std::string &sas_token, const unsigned int concurrency, const bool use_https, + const std::string &blob_endpoint) + { + if(account_name.empty() || ((account_key.empty() && sas_token.empty()) || (!account_key.empty() && !sas_token.empty()))) + { + errno = invalid_parameters; + return blob_client_wrapper(false); + } + + /* set a default concurrency value. */ + unsigned int concurrency_limit = 40; + if(concurrency != 0) + { + concurrency_limit = concurrency; + } + std::string accountName(account_name); + std::string accountKey(account_key); + + try + { + std::shared_ptr cred; + if (account_key.length() > 0) + { + cred = std::make_shared(accountName, accountKey); + } + else + { + // We have already verified that exactly one form of credentials is present, so if shared key is not present, it must be sas. + cred = std::make_shared(sas_token); + } + std::shared_ptr account = std::make_shared(accountName, cred, use_https, blob_endpoint); + std::shared_ptr blobClient= std::make_shared(account, concurrency_limit); + errno = 0; + return blob_client_wrapper(blobClient); + } + catch(const std::exception &ex) + { + syslog(LOG_ERR, "Failed to create blob client. ex.what() = %s.", ex.what()); + errno = unknown_error; + return blob_client_wrapper(false); + } + } + + void blob_client_wrapper::create_container(const std::string &container) + { + if(!is_valid()) + { + errno = client_not_init; + return; + } + if(container.empty()) + { + errno = invalid_parameters; + return; + } + + try + { + auto task = m_blobClient->create_container(container); + auto result = task.get(); + + if(!result.success()) + { + /* container already exists. + * * Bug, need to compare message as well. + * * */ + errno = std::stoi(result.error().code); + } + else + { + errno = 0; + } + } + catch(std::exception& ex) + { + syslog(LOG_ERR, "Unknown failure in create_container. ex.what() = %s, container = %s.", ex.what(), container.c_str()); + errno = unknown_error; + return; + } + } + + void blob_client_wrapper::delete_container(const std::string &container) + { + if(!is_valid()) + { + errno = client_not_init; + return; + } + if(container.empty()) + { + errno = invalid_parameters; + return; + } + + try + { + auto task = m_blobClient->delete_container(container); + auto result = task.get(); + + if(!result.success()) + { + errno = std::stoi(result.error().code); + } + else + { + errno = 0; + } + } + catch(std::exception& ex) + { + syslog(LOG_ERR, "Unknown failure in delete_container. ex.what() = %s, container = %s.", ex.what(), container.c_str()); + errno = unknown_error; + return; + } + } + + bool blob_client_wrapper::container_exists(const std::string &container) + { + if(!is_valid()) + { + errno = client_not_init; + return false; + } + if(container.empty()) + { + errno = invalid_parameters; + return false; + } + + try + { + auto containerProperty = m_blobClient->get_container_property(container).response(); + + if(containerProperty.valid()) + { + errno = 0; + return true; + } + else + { + syslog(LOG_ERR, "Unknown failure in container_exists. No exception, but the container property object is invalid. errno = %d.", errno); + errno = unknown_error; + return false; + } + } + catch(std::exception& ex) + { + syslog(LOG_ERR, "Unknown failure in container_exists. ex.what() = %s, container = %s.", ex.what(), container.c_str()); + errno = unknown_error; + return false; + } + } + + std::vector blob_client_wrapper::list_containers_segmented(const std::string &prefix, const std::string& continuation_token, const int max_result, bool include_metadata) + { + if(!is_valid()) + { + errno = client_not_init; + return std::vector(); + } + if(prefix.length() == 0) + { + errno = invalid_parameters; + return std::vector(); + } + + try + { + auto task = m_blobClient->list_containers_segmented(prefix, continuation_token, max_result, include_metadata); + auto result = task.get(); + + if(!result.success()) + { + errno = std::stoi(result.error().code); + return std::vector(); + } + return result.response().containers; + } + catch(std::exception& ex) + { + syslog(LOG_ERR, "Unknown failure in list_containers. ex.what() = %s, prefix = %s.", ex.what(), prefix.c_str()); + errno = unknown_error; + return std::vector(); + } + } + + list_blobs_segmented_response blob_client_wrapper::list_blobs_segmented(const std::string &container, const std::string &delimiter, const std::string &continuation_token, const std::string &prefix, int max_results) + { + if(!is_valid()) + { + errno = client_not_init; + return list_blobs_segmented_response(); + } + if(container.empty()) + { + errno = invalid_parameters; + return list_blobs_segmented_response(); + } + + try + { + auto task = m_blobClient->list_blobs_segmented(container, delimiter, continuation_token, prefix, max_results); + auto result = task.get(); + + if(!result.success()) + { + errno = std::stoi(result.error().code); + return list_blobs_segmented_response(); + } + else + { + errno = 0; + return result.response(); + } + } + catch(std::exception& ex) + { + syslog(LOG_ERR, "Unknown failure in list_blobs_hierarchial. ex.what() = %s, container = %s, prefix = %s.", ex.what(), container.c_str(), prefix.c_str()); + errno = unknown_error; + return list_blobs_segmented_response(); + } + } + + void blob_client_wrapper::put_blob(const std::string &sourcePath, const std::string &container, const std::string blob, const std::vector> &metadata) + { + if(!is_valid()) + { + errno = client_not_init; + return; + } + if(sourcePath.empty() || container.empty() || blob.empty()) + { + errno = invalid_parameters; + return; + } + + std::ifstream ifs; + try + { + ifs.open(sourcePath, std::ifstream::in); + } + catch(std::exception& ex) + { + // TODO open failed + syslog(LOG_ERR, "Failure to open the input stream in put_blob. ex.what() = %s, sourcePath = %s.", ex.what(), sourcePath.c_str()); + errno = unknown_error; + return; + } + + try + { + auto task = m_blobClient->upload_block_blob_from_stream(container, blob, ifs, metadata); + auto result = task.get(); + if(!result.success()) + { + errno = std::stoi(result.error().code); + } + else + { + errno = 0; + } + } + catch(std::exception& ex) + { + syslog(LOG_ERR, "Failure to upload the blob in put_blob. ex.what() = %s, container = %s, blob = %s, sourcePath = %s.", ex.what(), container.c_str(), blob.c_str(), sourcePath.c_str()); + errno = unknown_error; + } + + try + { + ifs.close(); + } + catch(std::exception& ex) + { + // TODO close failed + syslog(LOG_ERR, "Failure to close the input stream in put_blob. ex.what() = %s, container = %s, blob = %s, sourcePath = %s.", ex.what(), container.c_str(), blob.c_str(), sourcePath.c_str()); + errno = unknown_error; + } + } + + void blob_client_wrapper::upload_block_blob_from_stream(const std::string &container, const std::string blob, std::istream &is, const std::vector> &metadata) + { + if(!is_valid()) + { + errno = client_not_init; + return; + } + if(container.empty() || blob.empty()) + { + errno = invalid_parameters; + return; + } + + try + { + auto task = m_blobClient->upload_block_blob_from_stream(container, blob, is, metadata); + auto result = task.get(); + if(!result.success()) + { + errno = std::stoi(result.error().code); + if (errno == 0) { + errno = 503; + } + } + else + { + errno = 0; + } + } + catch(std::exception& ex) + { + syslog(LOG_ERR, "Unknown failure in upload_block_blob_from_stream. ex.what() = %s, container = %s, blob = %s.", ex.what(), container.c_str(), blob.c_str()); + errno = unknown_error; + } + } + + void blob_client_wrapper::upload_file_to_blob(const std::string &sourcePath, const std::string &container, const std::string blob, const std::vector> &metadata, size_t parallel) + { + if(!is_valid()) + { + errno = client_not_init; + return; + } + if(sourcePath.empty() || container.empty() || blob.empty()) + { + errno = invalid_parameters; + return; + } + + off_t fileSize = get_file_size(sourcePath.c_str()); + if(fileSize < 0) + { + /*errno already set by get_file_size*/ + return; + } + + if(fileSize <= 64*1024*1024) + { + put_blob(sourcePath, container, blob, metadata); + // put_blob sets errno + return; + } + + int result = 0; + + //support blobs up to 4.77TB = if file is larger, return EFBIG error + //need to round to the nearest multiple of 4MB for efficiency + if(fileSize > MAX_BLOB_SIZE) + { + errno = EFBIG; + return; + } + + long long block_size = MIN_UPLOAD_CHUNK_SIZE; + + if(fileSize > (50000 * MIN_UPLOAD_CHUNK_SIZE)) + { + long long min_block = fileSize / 50000; + int remainder = min_block % 4*1024*1024; + min_block += 4*1024*1024 - remainder; + block_size = min_block < MIN_UPLOAD_CHUNK_SIZE ? MIN_UPLOAD_CHUNK_SIZE : min_block; + } + + std::ifstream ifs(sourcePath); + if(!ifs) + { + syslog(LOG_ERR, "Failed to open the input stream in upload_file_to_blob. errno = %d, sourcePath = %s.", errno, sourcePath.c_str()); + errno = unknown_error; + return; + } + + std::vector block_list; + std::deque> task_list; + std::mutex mutex; + std::condition_variable cv; + std::mutex cv_mutex; + + for(long long offset = 0, idx = 0; offset < fileSize; offset += block_size, ++idx) + { + // control the number of submitted jobs. + while(task_list.size() > m_concurrency) + { + auto r = task_list.front().get(); + task_list.pop_front(); + if (0 == result) { + result = r; + } + } + if (0 != result) { + break; + } + int length = block_size; + if(offset + length > fileSize) + { + length = fileSize - offset; + } + + char* buffer = (char*)malloc(block_size); + if (!buffer) { + result = 12; + break; + } + if(!ifs.read(buffer, length)) + { + syslog(LOG_ERR, "Failed to read from input stream in upload_file_to_blob. sourcePath = %s, container = %s, blob = %s, offset = %lld, length = %d.", sourcePath.c_str(), container.c_str(), blob.c_str(), offset, length); + result = unknown_error; + break; + } + std::string raw_block_id = std::to_string(idx); + //pad the string to length of 6. + raw_block_id.insert(raw_block_id.begin(), 12 - raw_block_id.length(), '0'); + const std::string block_id(to_base64((raw_block_id + get_uuid()).c_str(), 64)); + put_block_list_request_base::block_item block; + block.id = block_id; + block.type = put_block_list_request_base::block_type::uncommitted; + block_list.push_back(block); + auto single_put = std::async(std::launch::async, [block_id, this, buffer, length, &container, &blob, ¶llel, &mutex, &cv_mutex, &cv](){ + { + std::unique_lock lk(cv_mutex); + cv.wait(lk, [¶llel, &mutex]() { + std::lock_guard lock(mutex); + if(parallel > 0) + { + --parallel; + return true; + } + return false; + }); + } + + std::istringstream in; + in.rdbuf()->pubsetbuf(buffer, length); + const auto blockResult = m_blobClient->upload_block_from_stream(container, blob, block_id, in).get(); + free(buffer); + + { + std::lock_guard lock(mutex); + ++parallel; + cv.notify_one(); + } + + int result = 0; + if(!blockResult.success()) + { + result = std::stoi(blockResult.error().code); + if (0 == result) { + // It seems that timeouted requests has no code setup + result = 503; + } + } + return result; + }); + task_list.push_back(std::move(single_put)); + } + + // wait for the rest of tasks + for(auto &task: task_list) + { + const auto r = task.get(); + if(0 == result) + { + result = r; + } + } + if (0 != result) { + } + if(result == 0) + { + const auto r = m_blobClient->put_block_list(container, blob, block_list, metadata).get(); + if(!r.success()) + { + result = std::stoi(r.error().code); + syslog(LOG_ERR, "put_block_list failed in upload_file_to_blob. error code = %d, sourcePath = %s, container = %s, blob = %s.", result, sourcePath.c_str(), container.c_str(), blob.c_str()); + if (0 == result) { + result = unknown_error; + } + } + } + + ifs.close(); + errno = result; + } + + off_t get_file_size(const char* path) + { + struct stat st; + if(stat(path, &st) == 0) + { + return st.st_size; + } + return -1; + } + + void blob_client_wrapper::download_blob_to_stream(const std::string &container, const std::string &blob, unsigned long long offset, unsigned long long size, std::ostream &os) + { + if(!is_valid()) + { + errno = client_not_init; + return; + } + + try + { + auto task = m_blobClient->download_blob_to_stream(container, blob, offset, size, os); + task.wait(); + auto result = task.get(); + + if(!result.success()) + { + errno = std::stoi(result.error().code); + } + else + { + errno = 0; + } + } + catch(std::exception& ex) + { + syslog(LOG_ERR, "Unknown failure in download_blob_to_stream. ex.what() = %s, container = %s, blob = %s.", ex.what(), container.c_str(), blob.c_str()); + errno = unknown_error; + return; + } + } + + void blob_client_wrapper::download_blob_to_file(const std::string &container, const std::string &blob, const std::string &destPath, time_t &returned_last_modified, size_t parallel) + { + if(!is_valid()) + { + errno = client_not_init; + return; + } + + const size_t downloaders = std::min(parallel, static_cast(m_concurrency)); + storage_outcome firstChunk; + try + { + // Download the first chunk of the blob. The response will contain required blob metadata as well. + int errcode = 0; + std::ofstream os(destPath.c_str(), std::ofstream::binary | std::ofstream::out); + firstChunk = m_blobClient->get_chunk_to_stream_sync(container, blob, 0, DOWNLOAD_CHUNK_SIZE, os); + os.close(); + if (!os) { + syslog(LOG_ERR, "get_chunk_to_stream_async failed for firstchunk in download_blob_to_file. container = %s, blob = %s, destPath = %s.", container.c_str(), blob.c_str(), destPath.c_str()); + errno = unknown_error; + return; + } + if (!firstChunk.success()) + { + if (constants::code_request_range_not_satisfiable != firstChunk.error().code) { + errno = std::stoi(firstChunk.error().code); + return; + } + // The only reason for constants::code_request_range_not_satisfiable on the first chunk is zero + // blob size, so proceed as there is no error. + } + // Smoke check if the total size is known, otherwise - fail. + if (firstChunk.response().totalSize < 0) { + errno = blob_no_content_range; + return; + } + + // Get required metadata - etag to verify all future chunks and the total blob size. + const auto originalEtag = firstChunk.response().etag; + const auto length = static_cast(firstChunk.response().totalSize); + + // Resize the target file. + auto fd = open(destPath.c_str(), O_WRONLY, 0770); + if (-1 == fd) { + return; + } + if (-1 == ftruncate(fd, length)) { + close(fd); + return; + } + close(fd); + + // Download the rest. + const auto left = length - firstChunk.response().size; + const auto chunk_size = std::max(DOWNLOAD_CHUNK_SIZE, (left + downloaders - 1)/ downloaders); + std::vector> task_list; + for(unsigned long long offset = firstChunk.response().size; offset < length; offset += chunk_size) + { + const auto range = std::min(chunk_size, length - offset); + auto single_download = std::async(std::launch::async, [originalEtag, offset, range, this, &destPath, &container, &blob](){ + // Note, keep std::ios_base::in to prevent truncating of the file. + std::ofstream output(destPath.c_str(), std::ios_base::out | std::ios_base::in); + output.seekp(offset); + auto chunk = m_blobClient->get_chunk_to_stream_sync(container, blob, offset, range, output); + output.close(); + if(!chunk.success()) + { + // Looks like the blob has been replaced by smaller one - ask user to retry. + if (constants::code_request_range_not_satisfiable == chunk.error().code) { + return EAGAIN; + } + return std::stoi(chunk.error().code); + } + // The etag has been changed - ask user to retry. + if (originalEtag != chunk.response().etag) { + return EAGAIN; + } + // Check for any writing errors. + if (!output) { + syslog(LOG_ERR, "get_chunk_to_stream_async failure in download_blob_to_file. container = %s, blob = %s, destPath = %s, offset = %llu, range = %llu.", container.c_str(), blob.c_str(), destPath.c_str(), offset, range); + return unknown_error; + } + return 0; + }); + task_list.push_back(std::move(single_download)); + } + + // Wait for workers to complete downloading. + for(size_t i = 0; i < task_list.size(); ++i) + { + task_list[i].wait(); + auto result = task_list[i].get(); + // let's report the first encountered error for consistency. + if (0 != result && errcode == 0) { + errcode = result; + } + } + errno = errcode; + } + catch(std::exception& ex) + { + syslog(LOG_ERR, "Unknown failure in download_blob_to_file. ex.what() = %s, container = %s, blob = %s, destPath = %s.", ex.what(), container.c_str(), blob.c_str(), destPath.c_str()); + errno = unknown_error; + return; + } + + returned_last_modified = firstChunk.response().last_modified; + return; + } + + blob_property blob_client_wrapper::get_blob_property(const std::string &container, const std::string &blob) + { + if(!is_valid()) + { + errno = client_not_init; + return blob_property(false); + } + + try + { + auto result = m_blobClient->get_blob_property(container, blob); + if(!result.success()) + { + errno = std::stoi(result.error().code); + return blob_property(false); + } + else + { + errno = 0; + return result.response(); + } + } + catch(std::exception& ex) + { + syslog(LOG_ERR, "Unknown failure in get_blob_property. ex.what() = %s, container = %s, blob = %s.", ex.what(), container.c_str(), blob.c_str()); + errno = unknown_error; + return blob_property(false); + } + } + + bool blob_client_wrapper::blob_exists(const std::string &container, const std::string &blob) + { + if(!is_valid()) + { + errno = client_not_init; + return false; + } + + try + { + auto blobProperty = get_blob_property(container, blob); + if(blobProperty.valid()) + { + errno = 0; + return true; + } + return false; + } + catch(std::exception& ex) + { + syslog(LOG_ERR, "Unknown failure in blob_exists. ex.what() = %s, container = %s, blob = %s.", ex.what(), container.c_str(), blob.c_str()); + errno = unknown_error; + return false; + } + } + + void blob_client_wrapper::delete_blob(const std::string &container, const std::string &blob) + { + if(!is_valid()) + { + errno = client_not_init; + return; + } + if(container.empty() || blob.empty()) + { + errno = invalid_parameters; + return; + } + + try + { + auto task = m_blobClient->delete_blob(container, blob); + task.wait(); + auto result = task.get(); + + if(!result.success()) + { + errno = std::stoi(result.error().code); + } + else + { + errno = 0; + } + } + catch(std::exception& ex) + { + syslog(LOG_ERR, "Unknown failure in delete_blob. ex.what() = %s, container = %s, blob = %s.", ex.what(), container.c_str(), blob.c_str()); + errno = unknown_error; + return; + } + } + + void blob_client_wrapper::start_copy(const std::string &sourceContainer, const std::string &sourceBlob, const std::string &destContainer, const std::string &destBlob) + { + + if(!is_valid()) + { + errno = client_not_init; + return; + } + if(sourceContainer.empty() || sourceBlob.empty() || + destContainer.empty() || destBlob.empty()) + { + errno = invalid_parameters; + return; + } + + try + { + auto task = m_blobClient->start_copy(sourceContainer, sourceBlob, destContainer, destBlob); + task.wait(); + auto result = task.get(); + + if(!result.success()) + { + errno = std::stoi(result.error().code); + } + else + { + errno = 0; + } + } + catch(std::exception& ex) + { + syslog(LOG_ERR, "Unknown failure in start_copy. ex.what() = %s, sourceContainer = %s, sourceBlob = %s, destContainer = %s, destBlob = %s.", ex.what(), sourceContainer.c_str(), sourceBlob.c_str(), destContainer.c_str(), destBlob.c_str()); + errno = unknown_error; + return; + } + } + +}} // azure::storage_lite diff --git a/src/constants.cpp b/src/constants.cpp new file mode 100644 index 0000000..5f4e7e0 --- /dev/null +++ b/src/constants.cpp @@ -0,0 +1,9 @@ +#include "constants.h" + +namespace azure { namespace storage_lite { namespace constants { + +#define DAT(x, y) const char *x{ y }; +#include "constants.dat" +#undef DAT + +}}} // azure::storage_lite diff --git a/src/copy_blob_request_base.cpp b/src/copy_blob_request_base.cpp new file mode 100644 index 0000000..ca0b6ff --- /dev/null +++ b/src/copy_blob_request_base.cpp @@ -0,0 +1,44 @@ +#include "copy_blob_request_base.h" + +#include "constants.h" +#include "utility.h" + +namespace azure { namespace storage_lite { + + void copy_blob_request_base::build_request(const storage_account &a, http_base &h) const + { + const auto &r = *this; + + h.set_absolute_timeout(5L); + + h.set_method(http_base::http_method::put); + + // source + storage_url source_url = a.get_url(storage_account::service::blob); + source_url.append_path(r.container()).append_path(r.blob()); + + // dest + storage_url url = a.get_url(storage_account::service::blob); + url.append_path(r.destContainer()).append_path(r.destBlob()); + + add_optional_query(url, constants::query_timeout, r.timeout()); + h.set_url(url.to_string()); + + storage_headers headers; + // TODO: support access condition. + //add_access_condition_headers(h, headers, r); + add_content_length(h, headers, 0); + + add_ms_header(h, headers, constants::header_ms_client_request_id, r.ms_client_request_id(), true); + + h.add_header(constants::header_user_agent, constants::header_value_user_agent); + add_ms_header(h, headers, constants::header_ms_date, get_ms_date(date_format::rfc_1123)); + add_ms_header(h, headers, constants::header_ms_version, constants::header_value_storage_version); + + // set copy src + add_ms_header(h, headers, constants::header_ms_copy_source, a.credential()->transform_url(source_url.get_domain() + source_url.get_path())); + + a.credential()->sign_request(r, h, url, headers); + } + +}} // azure::storage_lite diff --git a/src/create_container_request_base.cpp b/src/create_container_request_base.cpp new file mode 100644 index 0000000..d80ddfd --- /dev/null +++ b/src/create_container_request_base.cpp @@ -0,0 +1,48 @@ +#include "create_container_request_base.h" + +#include "constants.h" +#include "utility.h" + +namespace azure { namespace storage_lite { + + void create_container_request_base::build_request(const storage_account &a, http_base &h) const + { + const auto &r = *this; + + h.set_absolute_timeout(5L); + + h.set_method(http_base::http_method::put); + + storage_url url = a.get_url(storage_account::service::blob); + url.append_path(r.container()); + + url.add_query(constants::query_restype, constants::query_restype_container); + add_optional_query(url, constants::query_timeout, r.timeout()); + h.set_url(url.to_string()); + + storage_headers headers; + add_content_length(h, headers, 0); + add_ms_header(h, headers, constants::header_ms_client_request_id, r.ms_client_request_id(), true); + + switch (r.ms_blob_public_access()) + { + case create_container_request_base::blob_public_access::blob: + add_ms_header(h, headers, constants::header_ms_blob_public_access, constants::header_value_blob_public_access_blob); + break; + case create_container_request_base::blob_public_access::container: + add_ms_header(h, headers, constants::header_ms_blob_public_access, constants::header_value_blob_public_access_container); + break; + default: + break; + } + + //TODO: add ms-meta + + h.add_header(constants::header_user_agent, constants::header_value_user_agent); + add_ms_header(h, headers, constants::header_ms_date, get_ms_date(date_format::rfc_1123)); + add_ms_header(h, headers, constants::header_ms_version, constants::header_value_storage_version); + + a.credential()->sign_request(r, h, url, headers); + } + +}} diff --git a/src/delete_blob_request_base.cpp b/src/delete_blob_request_base.cpp new file mode 100644 index 0000000..fa98367 --- /dev/null +++ b/src/delete_blob_request_base.cpp @@ -0,0 +1,50 @@ +#include "delete_blob_request_base.h" + +#include "constants.h" +#include "utility.h" + +namespace azure { namespace storage_lite { + + void delete_blob_request_base::build_request(const storage_account &a, http_base &h) const { + const auto &r = *this; + + h.set_absolute_timeout(5L); + + h.set_method(http_base::http_method::del); + + storage_url url = a.get_url(storage_account::service::blob); + url.append_path(r.container()).append_path(r.blob()); + + add_optional_query(url, constants::query_snapshot, r.snapshot()); + add_optional_query(url, constants::query_timeout, r.timeout()); + h.set_url(url.to_string()); + + storage_headers headers; + add_access_condition_headers(h, headers, r); + + add_ms_header(h, headers, constants::header_ms_client_request_id, r.ms_client_request_id(), true); + add_ms_header(h, headers, constants::header_ms_lease_id, r.ms_lease_id(), true); + + if (r.snapshot().empty()) + { + switch (r.ms_delete_snapshots()) + { + case delete_blob_request_base::delete_snapshots::only: + add_ms_header(h, headers, constants::header_ms_delete_snapshots, constants::header_value_delete_snapshots_only); + break; + case delete_blob_request_base::delete_snapshots::include: + add_ms_header(h, headers, constants::header_ms_delete_snapshots, constants::header_value_delete_snapshots_include); + break; + default: + break; + } + } + + h.add_header(constants::header_user_agent, constants::header_value_user_agent); + add_ms_header(h, headers, constants::header_ms_date, get_ms_date(date_format::rfc_1123)); + add_ms_header(h, headers, constants::header_ms_version, constants::header_value_storage_version); + + a.credential()->sign_request(r, h, url, headers); + } + +}} // azure::storage_lite diff --git a/src/delete_container_request_base.cpp b/src/delete_container_request_base.cpp new file mode 100644 index 0000000..33ced7e --- /dev/null +++ b/src/delete_container_request_base.cpp @@ -0,0 +1,34 @@ +#include "delete_container_request_base.h" + +#include "constants.h" +#include "utility.h" + +namespace azure { namespace storage_lite { + + void delete_container_request_base::build_request(const storage_account &a, http_base &h) const + { + const auto &r = *this; + + h.set_absolute_timeout(5L); + + h.set_method(http_base::http_method::del); + + storage_url url = a.get_url(storage_account::service::blob); + url.append_path(r.container()); + + url.add_query(constants::query_restype, constants::query_restype_container); + add_optional_query(url, constants::query_timeout, r.timeout()); + h.set_url(url.to_string()); + + storage_headers headers; + add_ms_header(h, headers, constants::header_ms_client_request_id, r.ms_client_request_id(), true); + add_ms_header(h, headers, constants::header_ms_lease_id, r.ms_lease_id(), true); + + h.add_header(constants::header_user_agent, constants::header_value_user_agent); + add_ms_header(h, headers, constants::header_ms_date, get_ms_date(date_format::rfc_1123)); + add_ms_header(h, headers, constants::header_ms_version, constants::header_value_storage_version); + + a.credential()->sign_request(r, h, url, headers); + } + +}} // azure::storage_lite diff --git a/src/get_blob_property_request_base.cpp b/src/get_blob_property_request_base.cpp new file mode 100644 index 0000000..efe1b74 --- /dev/null +++ b/src/get_blob_property_request_base.cpp @@ -0,0 +1,36 @@ +#include "get_blob_property_request_base.h" + +#include "constants.h" +#include "utility.h" + +namespace azure { namespace storage_lite { + + void get_blob_property_request_base::build_request(const storage_account &a, http_base &h) const + { + const auto &r = *this; + + h.set_absolute_timeout(5L); + + h.set_method(http_base::http_method::head); + + storage_url url = a.get_url(storage_account::service::blob); + url.append_path(r.container()).append_path(r.blob()); + + add_optional_query(url, constants::query_snapshot, r.snapshot()); + add_optional_query(url, constants::query_timeout, r.timeout()); + h.set_url(url.to_string()); + + storage_headers headers; + + add_ms_header(h, headers, constants::header_ms_client_request_id, r.ms_client_request_id(), true); + // les is not supported. + // add_ms_header(h, headers, constants::header_ms_lease_id, r.ms_lease_id(), true); + + h.add_header(constants::header_user_agent, constants::header_value_user_agent); + add_ms_header(h, headers, constants::header_ms_date, get_ms_date(date_format::rfc_1123)); + add_ms_header(h, headers, constants::header_ms_version, constants::header_value_storage_version); + + a.credential()->sign_request(r, h, url, headers); + } + +}} // azure::storage_lite diff --git a/src/get_blob_request_base.cpp b/src/get_blob_request_base.cpp new file mode 100644 index 0000000..9e45fa5 --- /dev/null +++ b/src/get_blob_request_base.cpp @@ -0,0 +1,44 @@ +#include "get_blob_request_base.h" + +#include "constants.h" +#include "utility.h" + +namespace azure { namespace storage_lite { + + void get_blob_request_base::build_request(const storage_account &a, http_base &h) const + { + const auto &r = *this; + + h.set_data_rate_timeout(); + + h.set_method(http_base::http_method::get); + + storage_url url = a.get_url(storage_account::service::blob); + url.append_path(r.container()).append_path(r.blob()); + + add_optional_query(url, constants::query_snapshot, r.snapshot()); + add_optional_query(url, constants::query_timeout, r.timeout()); + h.set_url(url.to_string()); + + storage_headers headers; + add_access_condition_headers(h, headers, r); + + add_optional_header(h, constants::header_origin, r.origin()); + + add_ms_header(h, headers, constants::header_ms_client_request_id, r.ms_client_request_id(), true); + add_ms_header(h, headers, constants::header_ms_lease_id, r.ms_lease_id(), true); + add_ms_header(h, headers, constants::header_ms_range, get_ms_range(r.start_byte(), r.end_byte()), true); + if (r.ms_range_get_content_md5()) + { + // TODO check range + add_ms_header(h, headers, constants::header_ms_range_get_content_md5, "true"); + } + + h.add_header(constants::header_user_agent, constants::header_value_user_agent); + add_ms_header(h, headers, constants::header_ms_date, get_ms_date(date_format::rfc_1123)); + add_ms_header(h, headers, constants::header_ms_version, constants::header_value_storage_version); + + a.credential()->sign_request(r, h, url, headers); + } + +}} // azure::storage_lite diff --git a/src/get_block_list_request_base.cpp b/src/get_block_list_request_base.cpp new file mode 100644 index 0000000..032ba3e --- /dev/null +++ b/src/get_block_list_request_base.cpp @@ -0,0 +1,49 @@ +#include "get_block_list_request_base.h" + +#include "constants.h" +#include "utility.h" + +namespace azure { namespace storage_lite { + + void get_block_list_request_base::build_request(const storage_account &a, http_base &h) const + { + const auto &r = *this; + + // TODO: allow setting max execution time. + h.set_absolute_timeout(30L); + + h.set_method(http_base::http_method::get); + + storage_url url = a.get_url(storage_account::service::blob); + url.append_path(r.container()).append_path(r.blob()); + + url.add_query(constants::query_comp, constants::query_comp_blocklist); + add_optional_query(url, constants::query_snapshot, r.snapshot()); + switch (r.blocklisttype()) { + case blocklisttypes::all: + url.add_query(constants::query_blocklisttype, constants::query_blocklisttype_all); + break; + case blocklisttypes::uncommitted: + url.add_query(constants::query_blocklisttype, constants::query_blocklisttype_uncommitted); + break; + case blocklisttypes::committed: + url.add_query(constants::query_blocklisttype, constants::query_blocklisttype_committed); + break; + } + add_optional_query(url, constants::query_timeout, r.timeout()); + h.set_url(url.to_string()); + + storage_headers headers; + add_access_condition_headers(h, headers, r); + + add_ms_header(h, headers, constants::header_ms_client_request_id, r.ms_client_request_id(), true); + add_ms_header(h, headers, constants::header_ms_lease_id, r.ms_lease_id(), true); + + h.add_header(constants::header_user_agent, constants::header_value_user_agent); + add_ms_header(h, headers, constants::header_ms_date, get_ms_date(date_format::rfc_1123)); + add_ms_header(h, headers, constants::header_ms_version, constants::header_value_storage_version); + + a.credential()->sign_request(r, h, url, headers); + } + +}} diff --git a/src/get_container_property_request_base.cpp b/src/get_container_property_request_base.cpp new file mode 100644 index 0000000..6c2c86a --- /dev/null +++ b/src/get_container_property_request_base.cpp @@ -0,0 +1,35 @@ +#include "get_container_property_request_base.h" + +#include "constants.h" +#include "utility.h" + +namespace azure { namespace storage_lite { + + void get_container_property_request_base::build_request(const storage_account &a, http_base &h) const + { + const auto &r = *this; + + h.set_absolute_timeout(5L); + + h.set_method(http_base::http_method::head); + + storage_url url = a.get_url(storage_account::service::blob); + url.append_path(r.container()); + + url.add_query(constants::query_restype, constants::query_restype_container); + add_optional_query(url, constants::query_timeout, r.timeout()); + h.set_url(url.to_string()); + + storage_headers headers; + + add_ms_header(h, headers, constants::header_ms_client_request_id, r.ms_client_request_id(), true); + // les is not supported. + // add_ms_header(h, headers, constants::header_ms_lease_id, r.ms_lease_id(), true); + + h.add_header(constants::header_user_agent, constants::header_value_user_agent); + add_ms_header(h, headers, constants::header_ms_date, get_ms_date(date_format::rfc_1123)); + add_ms_header(h, headers, constants::header_ms_version, constants::header_value_storage_version); + + a.credential()->sign_request(r, h, url, headers); + } +}} diff --git a/src/get_page_ranges_request_base.cpp b/src/get_page_ranges_request_base.cpp new file mode 100644 index 0000000..7c578a9 --- /dev/null +++ b/src/get_page_ranges_request_base.cpp @@ -0,0 +1,38 @@ +#include "get_page_ranges_request_base.h" + +#include "constants.h" +#include "utility.h" + +namespace azure { namespace storage_lite { + + void get_page_ranges_request_base::build_request(const storage_account &a, http_base &h) const { + const auto &r = *this; + + h.set_absolute_timeout(30L); + + h.set_method(http_base::http_method::get); + + storage_url url = a.get_url(storage_account::service::blob); + url.append_path(r.container()).append_path(r.blob()); + + url.add_query(constants::query_comp, constants::query_comp_pagelist); + add_optional_query(url, constants::query_snapshot, r.snapshot()); + add_optional_query(url, constants::query_timeout, r.timeout()); + h.set_url(url.to_string()); + + storage_headers headers; + add_access_condition_headers(h, headers, r); + + add_ms_header(h, headers, constants::header_ms_range, get_ms_range(r.start_byte(), r.end_byte()), true); + + add_ms_header(h, headers, constants::header_ms_client_request_id, r.ms_client_request_id(), true); + add_ms_header(h, headers, constants::header_ms_lease_id, r.ms_lease_id(), true); + + h.add_header(constants::header_user_agent, constants::header_value_user_agent); + add_ms_header(h, headers, constants::header_ms_date, get_ms_date(date_format::rfc_1123)); + add_ms_header(h, headers, constants::header_ms_version, constants::header_value_storage_version); + + a.credential()->sign_request(r, h, url, headers); + } + +}} // azure::storage_lite diff --git a/src/hash.cpp b/src/hash.cpp new file mode 100644 index 0000000..30efacd --- /dev/null +++ b/src/hash.cpp @@ -0,0 +1,71 @@ +#include "hash.h" + +#include "base64.h" + +namespace azure { namespace storage_lite { +#ifdef _WIN32 + hmac_sha256_hash_algorithm hmac_sha256_hash_provider::_algorithm; + + std::string hmac_sha256_hash_provider::hash_impl(const std::string &input, const std::vector &key) + { + ULONG object_size = 0; + ULONG size_length = 0; + + NTSTATUS status = BCryptGetProperty(_algorithm.handle(), BCRYPT_OBJECT_LENGTH, (PUCHAR)&object_size, sizeof(ULONG), &size_length, 0); + if (status != 0) + { + throw std::system_error(status, std::system_category()); + } + std::vector hash_object(object_size); + + BCRYPT_HASH_HANDLE hash_handle; + status = BCryptCreateHash(_algorithm.handle(), &hash_handle, (PUCHAR)hash_object.data(), (ULONG)hash_object.size(), (PUCHAR)key.data(), (ULONG)key.size(), 0); + if (status != 0) + { + throw std::system_error(status, std::system_category()); + } + + status = BCryptHashData(hash_handle, (PUCHAR)input.data(), (ULONG)input.size(), 0); + if (status != 0) + { + throw std::system_error(status, std::system_category()); + } + + status = BCryptGetProperty(hash_handle, BCRYPT_HASH_LENGTH, (PUCHAR)&object_size, sizeof(ULONG), &size_length, 0); + if (status != 0) + { + throw std::system_error(status, std::system_category()); + } + std::vector hash(object_size); + + status = BCryptFinishHash(hash_handle, hash.data(), (ULONG)hash.size(), 0); + if (status != 0 && status != 0xc0000008) + { + throw std::system_error(status, std::system_category()); + } + + status = BCryptDestroyHash(hash_handle); + return to_base64(hash); + } +#else + std::string hash(const std::string &to_sign, const std::vector &key) + { + unsigned int l = SHA256_DIGEST_LENGTH; + unsigned char digest[SHA256_DIGEST_LENGTH]; + +#ifdef USE_OPENSSL + HMAC_CTX ctx; + HMAC_CTX_init(&ctx); + HMAC_Init_ex(&ctx, key.data(), key.size(), EVP_sha256(), NULL); + HMAC_Update(&ctx, (const unsigned char*)to_sign.c_str(), to_sign.size()); + HMAC_Final(&ctx, digest, &l); + HMAC_CTX_cleanup(&ctx); +#else + gnutls_hmac_fast(GNUTLS_MAC_SHA256, key.data(), key.size(), (const unsigned char *)to_sign.data(), to_sign.size(), digest); +#endif + + return to_base64(std::vector(digest, digest + l)); + } +#endif + +}} // azure::storage_lite diff --git a/src/http/libcurl_http_client.cpp b/src/http/libcurl_http_client.cpp new file mode 100644 index 0000000..260d6e1 --- /dev/null +++ b/src/http/libcurl_http_client.cpp @@ -0,0 +1,88 @@ +#include + +#include "http/libcurl_http_client.h" + +#include "constants.h" + +namespace azure { namespace storage_lite { + + CurlEasyRequest::CurlEasyRequest(std::shared_ptr client, CURL *h) + : m_client(client), + m_curl(h), + m_slist(NULL) + { + check_code(curl_easy_setopt(m_curl, CURLOPT_HEADERFUNCTION, header_callback)); + check_code(curl_easy_setopt(m_curl, CURLOPT_HEADERDATA, this)); + } + + CurlEasyRequest::~CurlEasyRequest() + { + curl_easy_reset(m_curl); + m_client->release_handle(m_curl); + if (m_slist) { + curl_slist_free_all(m_slist); + } + } + + CURLcode CurlEasyRequest::perform() + { + if (m_output_stream.valid()) + { + check_code(curl_easy_setopt(m_curl, CURLOPT_WRITEFUNCTION, write)); + check_code(curl_easy_setopt(m_curl, CURLOPT_WRITEDATA, this)); + } + check_code(curl_easy_setopt(m_curl, CURLOPT_CUSTOMREQUEST, NULL)); + switch (m_method) + { + case http_method::get: + check_code(curl_easy_setopt(m_curl, CURLOPT_HTTPGET, 1)); + break; + case http_method::put: + check_code(curl_easy_setopt(m_curl, CURLOPT_UPLOAD, 1)); + break; + case http_method::del: + check_code(curl_easy_setopt(m_curl, CURLOPT_CUSTOMREQUEST, constants::http_delete)); + break; + case http_method::head: + check_code(curl_easy_setopt(m_curl, CURLOPT_HTTPGET, 1L)); + check_code(curl_easy_setopt(m_curl, CURLOPT_NOBODY, 1L)); + break; + case http_method::post: + check_code(curl_easy_setopt(m_curl, CURLOPT_CUSTOMREQUEST, constants::http_post)); + break; + } + + check_code(curl_easy_setopt(m_curl, CURLOPT_URL, m_url.data())); + + m_slist = curl_slist_append(m_slist, "Transfer-Encoding:"); + m_slist = curl_slist_append(m_slist, "Expect:"); + check_code(curl_easy_setopt(m_curl, CURLOPT_HTTPHEADER, m_slist)); + + const auto result = curl_easy_perform(m_curl); + check_code(result); // has nothing to do with checks, just resets errno for succeeded ops. + return result; + } + + size_t CurlEasyRequest::header_callback(char *buffer, size_t size, size_t nitems, void *userdata) + { + CurlEasyRequest::REQUEST_TYPE *p = static_cast(userdata); + std::string header(buffer, size * nitems); + auto colon = header.find(':'); + if (colon == std::string::npos) { + auto space = header.find(' '); + if (space != std::string::npos) { + std::istringstream iss(header.substr(space)); + iss >> p->m_code; + if (p->m_switch_error_callback && (p->m_switch_error_callback)(p->m_code)) { + curl_easy_setopt(p->m_curl, CURLOPT_WRITEFUNCTION, error); + curl_easy_setopt(p->m_curl, CURLOPT_WRITEDATA, p); + } + } + } + else { + p->m_headers[header.substr(0, colon)] = header.substr(colon + 2); + } + return size * nitems; + } + +}} // azure::storage_lite diff --git a/src/list_blobs_request_base.cpp b/src/list_blobs_request_base.cpp new file mode 100644 index 0000000..6c8841d --- /dev/null +++ b/src/list_blobs_request_base.cpp @@ -0,0 +1,115 @@ +#include "list_blobs_request_base.h" + +#include "constants.h" +#include "utility.h" + +namespace azure { namespace storage_lite { + +void list_blobs_request_base::build_request(const storage_account &a, http_base &h) const +{ + const auto &r = *this; + + h.set_absolute_timeout(30L); + + h.set_method(http_base::http_method::get); + + storage_url url = a.get_url(storage_account::service::blob); + url.append_path(r.container()); + + url.add_query(constants::query_restype, constants::query_restype_container); + url.add_query(constants::query_comp, constants::query_comp_list); + add_optional_query(url, constants::query_prefix, r.prefix()); + add_optional_query(url, constants::query_delimiter, r.delimiter()); + add_optional_query(url, constants::query_marker, r.marker()); + add_optional_query(url, constants::query_maxresults, r.maxresults()); + std::string include(","); + if (r.includes() & list_blobs_request_base::include::snapshots) + { + include.append(",").append(constants::query_include_snapshots); + } + if (r.includes() & list_blobs_request_base::include::metadata) + { + include.append(",").append(constants::query_include_metadata); + } + if (r.includes() & list_blobs_request_base::include::uncommittedblobs) + { + include.append(",").append(constants::query_include_uncommittedblobs); + } + if (r.includes() & list_blobs_request_base::include::copy) + { + include.append(",").append(constants::query_include_copy); + } + add_optional_query(url, constants::query_include, include.substr(1)); + add_optional_query(url, constants::query_timeout, r.timeout()); + h.set_url(url.to_string()); + + storage_headers headers; + add_ms_header(h, headers, constants::header_ms_client_request_id, r.ms_client_request_id(), true); + + h.add_header(constants::header_user_agent, constants::header_value_user_agent); + add_ms_header(h, headers, constants::header_ms_date, get_ms_date(date_format::rfc_1123)); + add_ms_header(h, headers, constants::header_ms_version, constants::header_value_storage_version); + + a.credential()->sign_request(r, h, url, headers); +} + +void list_blobs_segmented_request_base::build_request(const storage_account &a, http_base &h) const +{ + const auto &r = *this; + + h.set_absolute_timeout(30L); + + h.set_method(http_base::http_method::get); + + storage_url url = a.get_url(storage_account::service::blob); + url.append_path(r.container()); + + url.add_query(constants::query_restype, constants::query_restype_container); + url.add_query(constants::query_comp, constants::query_comp_list); + if(r.prefix().length() > 0) + { + add_optional_query(url, constants::query_prefix, r.prefix()); + } + if(r.delimiter().length() > 0) + { + add_optional_query(url, constants::query_delimiter, r.delimiter()); + } + if(r.marker().length() > 0) + { + add_optional_query(url, constants::query_marker, r.marker()); + } + if(r.maxresults() != 0) + { + add_optional_query(url, constants::query_maxresults, r.maxresults()); + } + std::string include(""); + if (r.includes() & list_blobs_request_base::include::snapshots) + { + include.append(",").append(constants::query_include_snapshots); + } + if (r.includes() & list_blobs_request_base::include::metadata) + { + include.append(",").append(constants::query_include_metadata); + } + if (r.includes() & list_blobs_request_base::include::uncommittedblobs) + { + include.append(",").append(constants::query_include_uncommittedblobs); + } + if (r.includes() & list_blobs_request_base::include::copy) + { + include.append(",").append(constants::query_include_copy); + } + add_optional_query(url, constants::query_include, include.substr(1)); + add_optional_query(url, constants::query_timeout, r.timeout()); + h.set_url(url.to_string()); + + storage_headers headers; + add_ms_header(h, headers, constants::header_ms_client_request_id, r.ms_client_request_id(), true); + + h.add_header(constants::header_user_agent, constants::header_value_user_agent); + add_ms_header(h, headers, constants::header_ms_date, get_ms_date(date_format::rfc_1123)); + add_ms_header(h, headers, constants::header_ms_version, constants::header_value_storage_version); + + a.credential()->sign_request(r, h, url, headers); +} +}} // azure::storage_lite diff --git a/src/list_containers_request_base.cpp b/src/list_containers_request_base.cpp new file mode 100644 index 0000000..fd0bfb6 --- /dev/null +++ b/src/list_containers_request_base.cpp @@ -0,0 +1,39 @@ +#include "list_containers_request_base.h" + +#include "constants.h" +#include "utility.h" + +namespace azure { namespace storage_lite { + + void list_containers_request_base::build_request(const storage_account &a, http_base &h) const + { + const auto &r = *this; + + h.set_absolute_timeout(30L); + + h.set_method(http_base::http_method::get); + + storage_url url = a.get_url(storage_account::service::blob); + url.append_path(""); + + url.add_query(constants::query_comp, constants::query_comp_list); + add_optional_query(url, constants::query_prefix, r.prefix()); + add_optional_query(url, constants::query_marker, r.marker()); + add_optional_query(url, constants::query_maxresults, r.maxresults()); + if (r.include_metadata()) { + url.add_query(constants::query_include, constants::query_include_metadata); + } + add_optional_query(url, constants::query_timeout, r.timeout()); + h.set_url(url.to_string()); + + storage_headers headers; + add_ms_header(h, headers, constants::header_ms_client_request_id, r.ms_client_request_id(), true); + + h.add_header(constants::header_user_agent, constants::header_value_user_agent); + add_ms_header(h, headers, constants::header_ms_date, get_ms_date(date_format::rfc_1123)); + add_ms_header(h, headers, constants::header_ms_version, constants::header_value_storage_version); + + a.credential()->sign_request(r, h, url, headers); + } + +}} // azure::storage_lite diff --git a/src/put_blob_request_base.cpp b/src/put_blob_request_base.cpp new file mode 100644 index 0000000..5eff720 --- /dev/null +++ b/src/put_blob_request_base.cpp @@ -0,0 +1,79 @@ +#include "put_blob_request_base.h" + +#include "constants.h" +#include "utility.h" + +namespace azure { namespace storage_lite { + +void put_blob_request_base::build_request(const storage_account &a, http_base &h) const +{ + const auto &r = *this; + + h.set_data_rate_timeout(); + + h.set_method(http_base::http_method::put); + + storage_url url = a.get_url(storage_account::service::blob); + url.append_path(r.container()).append_path(r.blob()); + + add_optional_query(url, constants::query_timeout, r.timeout()); + h.set_url(url.to_string()); + + storage_headers headers; + add_optional_content_encoding(h, headers, r.content_encoding()); + add_optional_content_language(h, headers, r.content_language()); + add_content_length(h, headers, r.content_length()); + add_optional_content_md5(h, headers, r.content_md5()); + add_optional_content_type(h, headers, r.content_type()); + add_access_condition_headers(h, headers, r); + + add_optional_header(h, constants::header_cache_control, r.cache_control()); + add_optional_header(h, constants::header_origin, r.origin()); + + add_ms_header(h, headers, constants::header_ms_client_request_id, r.ms_client_request_id(), true); + add_ms_header(h, headers, constants::header_ms_lease_id, r.ms_lease_id(), true); + + add_ms_header(h, headers, constants::header_ms_blob_cache_control, r.ms_blob_cache_control(), true); + add_ms_header(h, headers, constants::header_ms_blob_content_disposition, r.ms_blob_content_disposition(), true); + add_ms_header(h, headers, constants::header_ms_blob_content_encoding, r.ms_blob_content_encoding(), true); + add_ms_header(h, headers, constants::header_ms_blob_content_language, r.ms_blob_content_language(), true); + if (r.ms_blob_type() == put_blob_request_base::blob_type::page_blob) + { + add_ms_header(h, headers, constants::header_ms_blob_content_length, std::to_string(r.ms_blob_content_length())); + } + add_ms_header(h, headers, constants::header_ms_blob_content_md5, r.ms_blob_content_md5(), true); + add_ms_header(h, headers, constants::header_ms_blob_content_type, r.ms_blob_content_type(), true); + if (r.ms_blob_type() == put_blob_request_base::blob_type::page_blob) + { + add_ms_header(h, headers, constants::header_ms_blob_sequence_number, std::to_string(r.ms_blob_sequence_number()), true); + } + + switch (r.ms_blob_type()) { + case put_blob_request_base::blob_type::block_blob: + add_ms_header(h, headers, constants::header_ms_blob_type, constants::header_value_blob_type_blockblob); + break; + case put_blob_request_base::blob_type::page_blob: + add_ms_header(h, headers, constants::header_ms_blob_type, constants::header_value_blob_type_pageblob); + break; + case put_blob_request_base::blob_type::append_blob: + add_ms_header(h, headers, constants::header_ms_blob_type, constants::header_value_blob_type_appendblob); + break; + } + + //add ms-meta + if (r.metadata().size() > 0) + { + for (unsigned int i = 0; i < r.metadata().size(); i++) + { + add_metadata_header(h, headers, r.metadata()[i].first, r.metadata()[i].second); + } + } + + h.add_header(constants::header_user_agent, constants::header_value_user_agent); + add_ms_header(h, headers, constants::header_ms_date, get_ms_date(date_format::rfc_1123)); + add_ms_header(h, headers, constants::header_ms_version, constants::header_value_storage_version); + + a.credential()->sign_request(r, h, url, headers); +} + +}} // azure::storage_lite diff --git a/src/put_block_list_request_base.cpp b/src/put_block_list_request_base.cpp new file mode 100644 index 0000000..88cbb0c --- /dev/null +++ b/src/put_block_list_request_base.cpp @@ -0,0 +1,60 @@ +#include "put_block_list_request_base.h" + +#include "constants.h" +#include "utility.h" +#include "xml_writer.h" +#include "storage_stream.h" + +namespace azure { namespace storage_lite { + +void put_block_list_request_base::build_request(const storage_account &a, http_base &h) const +{ + const auto &r = *this; + + h.set_absolute_timeout(30L); + + h.set_method(http_base::http_method::put); + + storage_url url = a.get_url(storage_account::service::blob); + url.append_path(r.container()).append_path(r.blob()); + + url.add_query(constants::query_comp, constants::query_comp_blocklist); + add_optional_query(url, constants::query_timeout, r.timeout()); + h.set_url(url.to_string()); + + auto xml = xml_writer::write_block_list(r.block_list()); + auto ss = std::make_shared(xml); + h.set_input_stream(storage_istream(ss)); + + storage_headers headers; + add_content_length(h, headers, static_cast(xml.size())); + add_optional_content_md5(h, headers, r.content_md5()); + add_access_condition_headers(h, headers, r); + + add_ms_header(h, headers, constants::header_ms_client_request_id, r.ms_client_request_id(), true); + add_ms_header(h, headers, constants::header_ms_lease_id, r.ms_lease_id(), true); + + add_ms_header(h, headers, constants::header_ms_blob_cache_control, r.ms_blob_cache_control(), true); + add_ms_header(h, headers, constants::header_ms_blob_content_disposition, r.ms_blob_content_disposition(), true); + add_ms_header(h, headers, constants::header_ms_blob_content_encoding, r.ms_blob_content_encoding(), true); + add_ms_header(h, headers, constants::header_ms_blob_content_language, r.ms_blob_content_language(), true); + add_ms_header(h, headers, constants::header_ms_blob_content_md5, r.ms_blob_content_md5(), true); + add_ms_header(h, headers, constants::header_ms_blob_content_type, r.ms_blob_content_type(), true); + + //add ms-meta + if (r.metadata().size() > 0) + { + for (unsigned int i = 0; i < r.metadata().size(); i++) + { + add_metadata_header(h, headers, r.metadata()[i].first, r.metadata()[i].second); + } + } + + h.add_header(constants::header_user_agent, constants::header_value_user_agent); + add_ms_header(h, headers, constants::header_ms_date, get_ms_date(date_format::rfc_1123)); + add_ms_header(h, headers, constants::header_ms_version, constants::header_value_storage_version); + + a.credential()->sign_request(r, h, url, headers); +} + +}} // azure::storage_lite diff --git a/src/put_block_request_base.cpp b/src/put_block_request_base.cpp new file mode 100644 index 0000000..d16cf13 --- /dev/null +++ b/src/put_block_request_base.cpp @@ -0,0 +1,36 @@ +#include "put_block_request_base.h" + +#include "constants.h" +#include "utility.h" + +namespace azure { namespace storage_lite { + + void put_block_request_base::build_request(const storage_account &a, http_base &h) const + { + const auto &r = *this; + + h.set_data_rate_timeout(); + + h.set_method(http_base::http_method::put); + + storage_url url = a.get_url(storage_account::service::blob); + url.append_path(r.container()).append_path(r.blob()); + + url.add_query(constants::query_comp, constants::query_comp_block); + url.add_query(constants::query_blockid, r.blockid()); + add_optional_query(url, constants::query_timeout, r.timeout()); + h.set_url(url.to_string()); + + storage_headers headers; + add_content_length(h, headers, r.content_length()); + + add_ms_header(h, headers, constants::header_ms_client_request_id, r.ms_client_request_id(), true); + + h.add_header(constants::header_user_agent, constants::header_value_user_agent); + add_ms_header(h, headers, constants::header_ms_date, get_ms_date(date_format::rfc_1123)); + add_ms_header(h, headers, constants::header_ms_version, constants::header_value_storage_version); + + a.credential()->sign_request(r, h, url, headers); + } + +}} diff --git a/src/put_page_request_base.cpp b/src/put_page_request_base.cpp new file mode 100644 index 0000000..88e3ce5 --- /dev/null +++ b/src/put_page_request_base.cpp @@ -0,0 +1,54 @@ +#include "put_page_request_base.h" + +#include "constants.h" +#include "utility.h" + +namespace azure { namespace storage_lite { + + void put_page_request_base::build_request(const storage_account &a, http_base &h) const + { + const auto &r = *this; + + h.set_data_rate_timeout(); + + h.set_method(http_base::http_method::put); + + storage_url url = a.get_url(storage_account::service::blob); + url.append_path(r.container()).append_path(r.blob()); + + url.add_query(constants::query_comp, constants::query_comp_page); + add_optional_query(url, constants::query_timeout, r.timeout()); + h.set_url(url.to_string()); + + storage_headers headers; + add_content_length(h, headers, r.content_length()); + add_optional_content_md5(h, headers, r.content_md5()); + add_access_condition_headers(h, headers, r); + + add_ms_header(h, headers, constants::header_ms_range, get_ms_range(r.start_byte(), r.end_byte()), true); + + switch (r.ms_page_write()) + { + case put_page_request_base::page_write::update: + add_ms_header(h, headers, constants::header_ms_page_write, constants::header_value_page_write_update); + break; + case put_page_request_base::page_write::clear: + add_ms_header(h, headers, constants::header_ms_page_write, constants::header_value_page_write_clear); + break; + } + + add_ms_header(h, headers, constants::header_ms_if_sequence_number_lt, r.ms_if_sequence_number_lt(), true); + add_ms_header(h, headers, constants::header_ms_if_sequence_number_le, r.ms_if_sequence_number_le(), true); + add_ms_header(h, headers, constants::header_ms_if_sequence_number_eq, r.ms_if_sequence_number_eq(), true); + + add_ms_header(h, headers, constants::header_ms_client_request_id, r.ms_client_request_id(), true); + add_ms_header(h, headers, constants::header_ms_lease_id, r.ms_lease_id(), true); + + h.add_header(constants::header_user_agent, constants::header_value_user_agent); + add_ms_header(h, headers, constants::header_ms_date, get_ms_date(date_format::rfc_1123)); + add_ms_header(h, headers, constants::header_ms_version, constants::header_value_storage_version); + + a.credential()->sign_request(r, h, url, headers); + } + +}} // azure::storage_lite diff --git a/src/storage_account.cpp b/src/storage_account.cpp new file mode 100644 index 0000000..d4fc79b --- /dev/null +++ b/src/storage_account.cpp @@ -0,0 +1,67 @@ +#include "storage_account.h" + +#include "constants.h" + +namespace azure { namespace storage_lite { + + // TODO: Clean up table queue and file services + storage_account::storage_account(const std::string &account_name, std::shared_ptr credential, bool use_https, const std::string &blob_endpoint) + : m_credential(credential) + { + if (use_https) + { + append_all("https://"); + } + else + { + append_all("http://"); + } + + if(blob_endpoint.empty()) + { + append_all(account_name); + + m_blob_domain.append(".blob"); + m_table_domain.append(".table"); + m_queue_domain.append(".queue"); + m_file_domain.append(".file"); + + append_all(constants::default_endpoint_suffix); + } + else + { + append_all(blob_endpoint); + } + } + + AZURE_STORAGE_API storage_url storage_account::get_url(service service) const + { + storage_url url; + switch (service) + { + case storage_account::service::blob: + url.set_domain(m_blob_domain); + break; + case storage_account::service::table: + url.set_domain(m_table_domain); + break; + case storage_account::service::queue: + url.set_domain(m_queue_domain); + break; + case storage_account::service::file: + url.set_domain(m_file_domain); + break; + } + + return url; + } + + AZURE_STORAGE_API void storage_account::append_all(const std::string &part) + { + m_blob_domain.append(part); + m_table_domain.append(part); + m_queue_domain.append(part); + m_file_domain.append(part); + } + +}} diff --git a/src/storage_credential.cpp b/src/storage_credential.cpp new file mode 100644 index 0000000..88c8448 --- /dev/null +++ b/src/storage_credential.cpp @@ -0,0 +1,87 @@ +#include "storage_credential.h" + +#include "base64.h" +#include "constants.h" +#include "hash.h" +#include "utility.h" + +namespace azure { namespace storage_lite { + + shared_key_credential::shared_key_credential(const std::string &account_name, const std::string &account_key) + : m_account_name(account_name), + m_account_key(from_base64(account_key)) {} + + shared_key_credential::shared_key_credential(const std::string &account_name, const std::vector &account_key) + : m_account_name(account_name), + m_account_key(account_key) {} + + void shared_key_credential::sign_request(const storage_request_base &, http_base &h, const storage_url &url, const storage_headers &headers) const + { + std::string string_to_sign(get_http_verb(h.get_method())); + string_to_sign.append("\n"); + + string_to_sign.append(headers.content_encoding).append("\n"); + string_to_sign.append(headers.content_language).append("\n"); + string_to_sign.append(headers.content_length).append("\n"); + string_to_sign.append(headers.content_md5).append("\n"); + string_to_sign.append(headers.content_type).append("\n"); + // TODO: add date to string_to_sign when date is supported. + string_to_sign.append("\n"); // Date + string_to_sign.append(headers.if_modified_since).append("\n"); + string_to_sign.append(headers.if_match).append("\n"); + string_to_sign.append(headers.if_none_match).append("\n"); + string_to_sign.append(headers.if_unmodified_since).append("\n"); + string_to_sign.append("\n"); // Range + + // Canonicalized headers + for (const auto &header : headers.ms_headers) + { + string_to_sign.append(header.first).append(":").append(header.second).append("\n"); + } + + // Canonicalized resource + string_to_sign.append("/").append(m_account_name).append(url.get_encoded_path()); + for (const auto &name : url.get_query()) { + string_to_sign.append("\n").append(name.first); + bool first_value = true; + for (const auto &value : name.second) { + if (first_value) { + string_to_sign.append(":"); + first_value = false; + } + else { + string_to_sign.append(","); + } + string_to_sign.append(value); + } + } + + std::string authorization("SharedKey "); +#ifdef _WIN32 + authorization.append(m_account_name).append(":").append(hmac_sha256_hash_provider::hash(string_to_sign, m_account_key)); +#else + authorization.append(m_account_name).append(":").append(hash(string_to_sign, m_account_key)); +#endif + h.add_header(constants::header_authorization, authorization); + } + + void shared_key_credential::sign_request(const table_request_base &, http_base &, const storage_url &, const storage_headers &) const {} + + std::string shared_access_signature_credential::transform_url(std::string url) const + { + if (url.find('?') != std::string::npos) { + url.append("&"); + } + else { + url.append("?"); + } + url.append(m_sas_token); + return url; + } + + void shared_access_signature_credential::sign_request(const storage_request_base &, http_base &h, const storage_url &, const storage_headers &) const + { + std::string transformed_url = transform_url(h.get_url()); + h.set_url(transformed_url); + } +}} // azure::storage_lite diff --git a/src/storage_url.cpp b/src/storage_url.cpp new file mode 100644 index 0000000..0fe73d0 --- /dev/null +++ b/src/storage_url.cpp @@ -0,0 +1,117 @@ +#include "storage_url.h" + +namespace azure { namespace storage_lite { + bool is_alnum(char ch) + { + return (ch >= 'A' && ch <= 'Z') || (ch >= 'a' && ch <= 'z') || (ch >= '0' && ch <= '9'); + } + + bool is_unreserved(char ch) + { + return is_alnum(ch) || ch == '-' || ch == '.' || ch == '_' || ch == '~'; + } + bool is_sub_delim(char ch) + { + switch (ch) + { + case '!': + case '$': + case '&': + case '\'': + case '(': + case ')': + case '*': + case '+': + case ',': + case ';': + case '=': + return true; + default: + return false; + } + } + + bool is_path_character(char ch) + { + return is_unreserved(ch) || is_sub_delim(ch) || ch == '%' || ch == '/' || ch == ':' || ch == '@'; + } + + bool is_query_character(char ch) + { + return is_path_character(ch) || ch == '?'; + } + + std::string encode_url_path(const std::string& path) + { + const char* const hex = "0123456789ABCDEF"; + std::string encoded; + for (size_t index = 0; index < path.size(); ++index) + { + char ch = path[index]; + if (!is_path_character(ch) + || ch == '%' + || ch == '+' + || ch == '&') + { + encoded.push_back('%'); + encoded.push_back(hex[(ch >> 4) & 0xF]); + encoded.push_back(hex[ch & 0xF]); + } + else + { + encoded.push_back(ch); + } + } + return encoded; + } + + std::string encode_url_query(const std::string& path) + { + const char* const hex = "0123456789ABCDEF"; + std::string encoded; + for (size_t index = 0; index < path.size(); ++index) + { + char ch = path[index]; + if (!is_query_character(ch) + || ch == '%' + || ch == '+' + || ch == '&') + { + encoded.push_back('%'); + encoded.push_back(hex[(ch >> 4) & 0xF]); + encoded.push_back(hex[ch & 0xF]); + } + else + { + encoded.push_back(ch); + } + } + return encoded; + } + + std::string storage_url::to_string() const + { + std::string url(m_domain); + url.append(encode_url_path(m_path)); + + bool first_query = true; + for (const auto &q : m_query) + { + if (first_query) + { + url.append("?"); + first_query = false; + } + else + { + url.append("&"); + } + for (const auto &value : q.second) + { + url.append(encode_url_query(q.first)).append("=").append(encode_url_query(value)); + } + } + return url; + } + +}} // azure::storage_lite diff --git a/src/tinyxml2.cpp b/src/tinyxml2.cpp new file mode 100644 index 0000000..733942b --- /dev/null +++ b/src/tinyxml2.cpp @@ -0,0 +1,2344 @@ +/* +Original code by Lee Thomason (www.grinninglizard.com) + +This software is provided 'as-is', without any express or implied +warranty. In no event will the authors be held liable for any +damages arising from the use of this software. + +Permission is granted to anyone to use this software for any +purpose, including commercial applications, and to alter it and +redistribute it freely, subject to the following restrictions: + +1. The origin of this software must not be misrepresented; you must +not claim that you wrote the original software. If you use this +software in a product, an acknowledgment in the product documentation +would be appreciated but is not required. + +2. Altered source versions must be plainly marked as such, and +must not be misrepresented as being the original software. + +3. This notice may not be removed or altered from any source +distribution. +*/ + +#include "tinyxml2.h" + +#include // yes, this one new style header, is in the Android SDK. +#if defined(ANDROID_NDK) || defined(__QNXNTO__) +# include +#else +# include +#endif + +static const char LINE_FEED = (char)0x0a; // all line endings are normalized to LF +static const char LF = LINE_FEED; +static const char CARRIAGE_RETURN = (char)0x0d; // CR gets filtered out +static const char CR = CARRIAGE_RETURN; +static const char SINGLE_QUOTE = '\''; +static const char DOUBLE_QUOTE = '\"'; + +// Bunch of unicode info at: +// http://www.unicode.org/faq/utf_bom.html +// ef bb bf (Microsoft "lead bytes") - designates UTF-8 + +static const unsigned char TIXML_UTF_LEAD_0 = 0xefU; +static const unsigned char TIXML_UTF_LEAD_1 = 0xbbU; +static const unsigned char TIXML_UTF_LEAD_2 = 0xbfU; + +namespace tinyxml2 +{ + +struct Entity { + const char* pattern; + int length; + char value; +}; + +static const int NUM_ENTITIES = 5; +static const Entity entities[NUM_ENTITIES] = { + { "quot", 4, DOUBLE_QUOTE }, + { "amp", 3, '&' }, + { "apos", 4, SINGLE_QUOTE }, + { "lt", 2, '<' }, + { "gt", 2, '>' } +}; + + +StrPair::~StrPair() +{ + Reset(); +} + + +void StrPair::TransferTo( StrPair* other ) +{ + if ( this == other ) { + return; + } + // This in effect implements the assignment operator by "moving" + // ownership (as in auto_ptr). + + TIXMLASSERT( other->_flags == 0 ); + TIXMLASSERT( other->_start == 0 ); + TIXMLASSERT( other->_end == 0 ); + + other->Reset(); + + other->_flags = _flags; + other->_start = _start; + other->_end = _end; + + _flags = 0; + _start = 0; + _end = 0; +} + +void StrPair::Reset() +{ + if ( _flags & NEEDS_DELETE ) { + delete [] _start; + } + _flags = 0; + _start = 0; + _end = 0; +} + + +void StrPair::SetStr( const char* str, int flags ) +{ + Reset(); + size_t len = strlen( str ); + _start = new char[ len+1 ]; + memcpy( _start, str, len+1 ); + _end = _start + len; + _flags = flags | NEEDS_DELETE; +} + + +char* StrPair::ParseText( char* p, const char* endTag, int strFlags ) +{ + TIXMLASSERT( endTag && *endTag ); + + char* start = p; + char endChar = *endTag; + size_t length = strlen( endTag ); + + // Inner loop of text parsing. + while ( *p ) { + if ( *p == endChar && strncmp( p, endTag, length ) == 0 ) { + Set( start, p, strFlags ); + return p + length; + } + ++p; + } + return 0; +} + + +char* StrPair::ParseName( char* p ) +{ + if ( !p || !(*p) ) { + return 0; + } + if ( !XMLUtil::IsNameStartChar( *p ) ) { + return 0; + } + + char* const start = p; + ++p; + while ( *p && XMLUtil::IsNameChar( *p ) ) { + ++p; + } + + Set( start, p, 0 ); + return p; +} + + +void StrPair::CollapseWhitespace() +{ + // Adjusting _start would cause undefined behavior on delete[] + TIXMLASSERT( ( _flags & NEEDS_DELETE ) == 0 ); + // Trim leading space. + _start = XMLUtil::SkipWhiteSpace( _start ); + + if ( *_start ) { + char* p = _start; // the read pointer + char* q = _start; // the write pointer + + while( *p ) { + if ( XMLUtil::IsWhiteSpace( *p )) { + p = XMLUtil::SkipWhiteSpace( p ); + if ( *p == 0 ) { + break; // don't write to q; this trims the trailing space. + } + *q = ' '; + ++q; + } + *q = *p; + ++q; + ++p; + } + *q = 0; + } +} + + +const char* StrPair::GetStr() +{ + TIXMLASSERT( _start ); + TIXMLASSERT( _end ); + if ( _flags & NEEDS_FLUSH ) { + *_end = 0; + _flags ^= NEEDS_FLUSH; + + if ( _flags ) { + char* p = _start; // the read pointer + char* q = _start; // the write pointer + + while( p < _end ) { + if ( (_flags & NEEDS_NEWLINE_NORMALIZATION) && *p == CR ) { + // CR-LF pair becomes LF + // CR alone becomes LF + // LF-CR becomes LF + if ( *(p+1) == LF ) { + p += 2; + } + else { + ++p; + } + *q++ = LF; + } + else if ( (_flags & NEEDS_NEWLINE_NORMALIZATION) && *p == LF ) { + if ( *(p+1) == CR ) { + p += 2; + } + else { + ++p; + } + *q++ = LF; + } + else if ( (_flags & NEEDS_ENTITY_PROCESSING) && *p == '&' ) { + // Entities handled by tinyXML2: + // - special entities in the entity table [in/out] + // - numeric character reference [in] + // 中 or 中 + + if ( *(p+1) == '#' ) { + const int buflen = 10; + char buf[buflen] = { 0 }; + int len = 0; + char* adjusted = const_cast( XMLUtil::GetCharacterRef( p, buf, &len ) ); + if ( adjusted == 0 ) { + *q = *p; + ++p; + ++q; + } + else { + TIXMLASSERT( 0 <= len && len <= buflen ); + TIXMLASSERT( q + len <= adjusted ); + p = adjusted; + memcpy( q, buf, len ); + q += len; + } + } + else { + int i=0; + for(; i(p); + // Check for BOM: + if ( *(pu+0) == TIXML_UTF_LEAD_0 + && *(pu+1) == TIXML_UTF_LEAD_1 + && *(pu+2) == TIXML_UTF_LEAD_2 ) { + *bom = true; + p += 3; + } + TIXMLASSERT( p ); + return p; +} + + +void XMLUtil::ConvertUTF32ToUTF8( unsigned long input, char* output, int* length ) +{ + const unsigned long BYTE_MASK = 0xBF; + const unsigned long BYTE_MARK = 0x80; + const unsigned long FIRST_BYTE_MARK[7] = { 0x00, 0x00, 0xC0, 0xE0, 0xF0, 0xF8, 0xFC }; + + if (input < 0x80) { + *length = 1; + } + else if ( input < 0x800 ) { + *length = 2; + } + else if ( input < 0x10000 ) { + *length = 3; + } + else if ( input < 0x200000 ) { + *length = 4; + } + else { + *length = 0; // This code won't covert this correctly anyway. + return; + } + + output += *length; + + // Scary scary fall throughs. + switch (*length) { + case 4: + --output; + *output = (char)((input | BYTE_MARK) & BYTE_MASK); + input >>= 6; /* fall-thru */ + case 3: + --output; + *output = (char)((input | BYTE_MARK) & BYTE_MASK); + input >>= 6; /* fall-thru */ + case 2: + --output; + *output = (char)((input | BYTE_MARK) & BYTE_MASK); + input >>= 6; /* fall-thru */ + case 1: + --output; + *output = (char)(input | FIRST_BYTE_MARK[*length]); + break; + default: + TIXMLASSERT( false ); + } +} + + +const char* XMLUtil::GetCharacterRef( const char* p, char* value, int* length ) +{ + // Presume an entity, and pull it out. + *length = 0; + + if ( *(p+1) == '#' && *(p+2) ) { + unsigned long ucs = 0; + TIXMLASSERT( sizeof( ucs ) >= 4 ); + ptrdiff_t delta = 0; + unsigned mult = 1; + static const char SEMICOLON = ';'; + + if ( *(p+2) == 'x' ) { + // Hexadecimal. + const char* q = p+3; + if ( !(*q) ) { + return 0; + } + + q = strchr( q, SEMICOLON ); + + if ( !q ) { + return 0; + } + TIXMLASSERT( *q == SEMICOLON ); + + delta = q-p; + --q; + + while ( *q != 'x' ) { + unsigned int digit = 0; + + if ( *q >= '0' && *q <= '9' ) { + digit = *q - '0'; + } + else if ( *q >= 'a' && *q <= 'f' ) { + digit = *q - 'a' + 10; + } + else if ( *q >= 'A' && *q <= 'F' ) { + digit = *q - 'A' + 10; + } + else { + return 0; + } + TIXMLASSERT( digit == 0 || mult <= UINT_MAX / digit ); + TIXMLASSERT( digit >= 0 && digit < 16); + const unsigned int digitScaled = mult * digit; + TIXMLASSERT( ucs <= ULONG_MAX - digitScaled ); + ucs += digitScaled; + TIXMLASSERT( mult <= UINT_MAX / 16 ); + mult *= 16; + --q; + } + } + else { + // Decimal. + const char* q = p+2; + if ( !(*q) ) { + return 0; + } + + q = strchr( q, SEMICOLON ); + + if ( !q ) { + return 0; + } + TIXMLASSERT( *q == SEMICOLON ); + + delta = q-p; + --q; + + while ( *q != '#' ) { + if ( *q >= '0' && *q <= '9' ) { + const unsigned int digit = *q - '0'; + TIXMLASSERT( digit == 0 || mult <= UINT_MAX / digit ); + const unsigned int digitScaled = mult * digit; + TIXMLASSERT( ucs <= ULONG_MAX - digitScaled ); + ucs += digitScaled; + } + else { + return 0; + } + TIXMLASSERT( mult <= UINT_MAX / 10 ); + mult *= 10; + --q; + } + } + // convert the UCS to UTF-8 + ConvertUTF32ToUTF8( ucs, value, length ); + return p + delta + 1; + } + return p+1; +} + + +void XMLUtil::ToStr( int v, char* buffer, int bufferSize ) +{ + TIXML_SNPRINTF( buffer, bufferSize, "%d", v ); +} + + +void XMLUtil::ToStr( unsigned v, char* buffer, int bufferSize ) +{ + TIXML_SNPRINTF( buffer, bufferSize, "%u", v ); +} + + +void XMLUtil::ToStr( bool v, char* buffer, int bufferSize ) +{ + TIXML_SNPRINTF( buffer, bufferSize, "%d", v ? 1 : 0 ); +} + +/* + ToStr() of a number is a very tricky topic. + https://github.com/leethomason/tinyxml2/issues/106 +*/ +void XMLUtil::ToStr( float v, char* buffer, int bufferSize ) +{ + TIXML_SNPRINTF( buffer, bufferSize, "%.8g", v ); +} + + +void XMLUtil::ToStr( double v, char* buffer, int bufferSize ) +{ + TIXML_SNPRINTF( buffer, bufferSize, "%.17g", v ); +} + + +bool XMLUtil::ToInt( const char* str, int* value ) +{ + if ( TIXML_SSCANF( str, "%d", value ) == 1 ) { + return true; + } + return false; +} + +bool XMLUtil::ToUnsigned( const char* str, unsigned *value ) +{ + if ( TIXML_SSCANF( str, "%u", value ) == 1 ) { + return true; + } + return false; +} + +bool XMLUtil::ToBool( const char* str, bool* value ) +{ + int ival = 0; + if ( ToInt( str, &ival )) { + *value = (ival==0) ? false : true; + return true; + } + if ( StringEqual( str, "true" ) ) { + *value = true; + return true; + } + else if ( StringEqual( str, "false" ) ) { + *value = false; + return true; + } + return false; +} + + +bool XMLUtil::ToFloat( const char* str, float* value ) +{ + if ( TIXML_SSCANF( str, "%f", value ) == 1 ) { + return true; + } + return false; +} + +bool XMLUtil::ToDouble( const char* str, double* value ) +{ + if ( TIXML_SSCANF( str, "%lf", value ) == 1 ) { + return true; + } + return false; +} + + +char* XMLDocument::Identify( char* p, XMLNode** node ) +{ + TIXMLASSERT( node ); + TIXMLASSERT( p ); + char* const start = p; + p = XMLUtil::SkipWhiteSpace( p ); + if( !*p ) { + *node = 0; + TIXMLASSERT( p ); + return p; + } + + // What is this thing? + // These strings define the matching patters: + static const char* xmlHeader = { "_memPool = &_commentPool; + p += xmlHeaderLen; + } + else if ( XMLUtil::StringEqual( p, commentHeader, commentHeaderLen ) ) { + TIXMLASSERT( sizeof( XMLComment ) == _commentPool.ItemSize() ); + returnNode = new (_commentPool.Alloc()) XMLComment( this ); + returnNode->_memPool = &_commentPool; + p += commentHeaderLen; + } + else if ( XMLUtil::StringEqual( p, cdataHeader, cdataHeaderLen ) ) { + TIXMLASSERT( sizeof( XMLText ) == _textPool.ItemSize() ); + XMLText* text = new (_textPool.Alloc()) XMLText( this ); + returnNode = text; + returnNode->_memPool = &_textPool; + p += cdataHeaderLen; + text->SetCData( true ); + } + else if ( XMLUtil::StringEqual( p, dtdHeader, dtdHeaderLen ) ) { + TIXMLASSERT( sizeof( XMLUnknown ) == _commentPool.ItemSize() ); + returnNode = new (_commentPool.Alloc()) XMLUnknown( this ); + returnNode->_memPool = &_commentPool; + p += dtdHeaderLen; + } + else if ( XMLUtil::StringEqual( p, elementHeader, elementHeaderLen ) ) { + TIXMLASSERT( sizeof( XMLElement ) == _elementPool.ItemSize() ); + returnNode = new (_elementPool.Alloc()) XMLElement( this ); + returnNode->_memPool = &_elementPool; + p += elementHeaderLen; + } + else { + TIXMLASSERT( sizeof( XMLText ) == _textPool.ItemSize() ); + returnNode = new (_textPool.Alloc()) XMLText( this ); + returnNode->_memPool = &_textPool; + p = start; // Back it up, all the text counts. + } + + TIXMLASSERT( returnNode ); + TIXMLASSERT( p ); + *node = returnNode; + return p; +} + + +bool XMLDocument::Accept( XMLVisitor* visitor ) const +{ + TIXMLASSERT( visitor ); + if ( visitor->VisitEnter( *this ) ) { + for ( const XMLNode* node=FirstChild(); node; node=node->NextSibling() ) { + if ( !node->Accept( visitor ) ) { + break; + } + } + } + return visitor->VisitExit( *this ); +} + + +// --------- XMLNode ----------- // + +XMLNode::XMLNode( XMLDocument* doc ) : + _document( doc ), + _parent( 0 ), + _firstChild( 0 ), _lastChild( 0 ), + _prev( 0 ), _next( 0 ), + _memPool( 0 ) +{ +} + + +XMLNode::~XMLNode() +{ + DeleteChildren(); + if ( _parent ) { + _parent->Unlink( this ); + } +} + +const char* XMLNode::Value() const +{ + return _value.GetStr(); +} + +void XMLNode::SetValue( const char* str, bool staticMem ) +{ + if ( staticMem ) { + _value.SetInternedStr( str ); + } + else { + _value.SetStr( str ); + } +} + + +void XMLNode::DeleteChildren() +{ + while( _firstChild ) { + TIXMLASSERT( _firstChild->_document == _document ); + XMLNode* node = _firstChild; + Unlink( node ); + + DeleteNode( node ); + } + _firstChild = _lastChild = 0; +} + + +void XMLNode::Unlink( XMLNode* child ) +{ + TIXMLASSERT( child ); + TIXMLASSERT( child->_document == _document ); + if ( child == _firstChild ) { + _firstChild = _firstChild->_next; + } + if ( child == _lastChild ) { + _lastChild = _lastChild->_prev; + } + + if ( child->_prev ) { + child->_prev->_next = child->_next; + } + if ( child->_next ) { + child->_next->_prev = child->_prev; + } + child->_parent = 0; +} + + +void XMLNode::DeleteChild( XMLNode* node ) +{ + TIXMLASSERT( node ); + TIXMLASSERT( node->_document == _document ); + TIXMLASSERT( node->_parent == this ); + DeleteNode( node ); +} + + +XMLNode* XMLNode::InsertEndChild( XMLNode* addThis ) +{ + TIXMLASSERT( addThis ); + if ( addThis->_document != _document ) { + TIXMLASSERT( false ); + return 0; + } + InsertChildPreamble( addThis ); + + if ( _lastChild ) { + TIXMLASSERT( _firstChild ); + TIXMLASSERT( _lastChild->_next == 0 ); + _lastChild->_next = addThis; + addThis->_prev = _lastChild; + _lastChild = addThis; + + addThis->_next = 0; + } + else { + TIXMLASSERT( _firstChild == 0 ); + _firstChild = _lastChild = addThis; + + addThis->_prev = 0; + addThis->_next = 0; + } + addThis->_parent = this; + return addThis; +} + + +XMLNode* XMLNode::InsertFirstChild( XMLNode* addThis ) +{ + TIXMLASSERT( addThis ); + if ( addThis->_document != _document ) { + TIXMLASSERT( false ); + return 0; + } + InsertChildPreamble( addThis ); + + if ( _firstChild ) { + TIXMLASSERT( _lastChild ); + TIXMLASSERT( _firstChild->_prev == 0 ); + + _firstChild->_prev = addThis; + addThis->_next = _firstChild; + _firstChild = addThis; + + addThis->_prev = 0; + } + else { + TIXMLASSERT( _lastChild == 0 ); + _firstChild = _lastChild = addThis; + + addThis->_prev = 0; + addThis->_next = 0; + } + addThis->_parent = this; + return addThis; +} + + +XMLNode* XMLNode::InsertAfterChild( XMLNode* afterThis, XMLNode* addThis ) +{ + TIXMLASSERT( addThis ); + if ( addThis->_document != _document ) { + TIXMLASSERT( false ); + return 0; + } + + TIXMLASSERT( afterThis ); + + if ( afterThis->_parent != this ) { + TIXMLASSERT( false ); + return 0; + } + + if ( afterThis->_next == 0 ) { + // The last node or the only node. + return InsertEndChild( addThis ); + } + InsertChildPreamble( addThis ); + addThis->_prev = afterThis; + addThis->_next = afterThis->_next; + afterThis->_next->_prev = addThis; + afterThis->_next = addThis; + addThis->_parent = this; + return addThis; +} + + + + +const XMLElement* XMLNode::FirstChildElement( const char* value ) const +{ + for( XMLNode* node=_firstChild; node; node=node->_next ) { + XMLElement* element = node->ToElement(); + if ( element ) { + if ( !value || XMLUtil::StringEqual( element->Name(), value ) ) { + return element; + } + } + } + return 0; +} + + +const XMLElement* XMLNode::LastChildElement( const char* value ) const +{ + for( XMLNode* node=_lastChild; node; node=node->_prev ) { + XMLElement* element = node->ToElement(); + if ( element ) { + if ( !value || XMLUtil::StringEqual( element->Name(), value ) ) { + return element; + } + } + } + return 0; +} + + +const XMLElement* XMLNode::NextSiblingElement( const char* value ) const +{ + for( XMLNode* node=this->_next; node; node = node->_next ) { + const XMLElement* element = node->ToElement(); + if ( element + && (!value || XMLUtil::StringEqual( value, node->Value() ))) { + return element; + } + } + return 0; +} + + +const XMLElement* XMLNode::PreviousSiblingElement( const char* value ) const +{ + for( XMLNode* node=_prev; node; node = node->_prev ) { + const XMLElement* element = node->ToElement(); + if ( element + && (!value || XMLUtil::StringEqual( value, node->Value() ))) { + return element; + } + } + return 0; +} + + +char* XMLNode::ParseDeep( char* p, StrPair* parentEnd ) +{ + // This is a recursive method, but thinking about it "at the current level" + // it is a pretty simple flat list: + // + // + // + // With a special case: + // + // + // + // + // Where the closing element (/foo) *must* be the next thing after the opening + // element, and the names must match. BUT the tricky bit is that the closing + // element will be read by the child. + // + // 'endTag' is the end tag for this node, it is returned by a call to a child. + // 'parentEnd' is the end tag for the parent, which is filled in and returned. + + while( p && *p ) { + XMLNode* node = 0; + + p = _document->Identify( p, &node ); + if ( node == 0 ) { + break; + } + + StrPair endTag; + p = node->ParseDeep( p, &endTag ); + if ( !p ) { + DeleteNode( node ); + if ( !_document->Error() ) { + _document->SetError( XML_ERROR_PARSING, 0, 0 ); + } + break; + } + + XMLElement* ele = node->ToElement(); + if ( ele ) { + // We read the end tag. Return it to the parent. + if ( ele->ClosingType() == XMLElement::CLOSING ) { + if ( parentEnd ) { + ele->_value.TransferTo( parentEnd ); + } + node->_memPool->SetTracked(); // created and then immediately deleted. + DeleteNode( node ); + return p; + } + + // Handle an end tag returned to this level. + // And handle a bunch of annoying errors. + bool mismatch = false; + if ( endTag.Empty() ) { + if ( ele->ClosingType() == XMLElement::OPEN ) { + mismatch = true; + } + } + else { + if ( ele->ClosingType() != XMLElement::OPEN ) { + mismatch = true; + } + else if ( !XMLUtil::StringEqual( endTag.GetStr(), node->Value() ) ) { + mismatch = true; + } + } + if ( mismatch ) { + _document->SetError( XML_ERROR_MISMATCHED_ELEMENT, node->Value(), 0 ); + DeleteNode( node ); + break; + } + } + InsertEndChild( node ); + } + return 0; +} + +void XMLNode::DeleteNode( XMLNode* node ) +{ + if ( node == 0 ) { + return; + } + MemPool* pool = node->_memPool; + node->~XMLNode(); + pool->Free( node ); +} + +void XMLNode::InsertChildPreamble( XMLNode* insertThis ) const +{ + TIXMLASSERT( insertThis ); + TIXMLASSERT( insertThis->_document == _document ); + + if ( insertThis->_parent ) + insertThis->_parent->Unlink( insertThis ); + else + insertThis->_memPool->SetTracked(); +} + +// --------- XMLText ---------- // +char* XMLText::ParseDeep( char* p, StrPair* ) +{ + const char* start = p; + if ( this->CData() ) { + p = _value.ParseText( p, "]]>", StrPair::NEEDS_NEWLINE_NORMALIZATION ); + if ( !p ) { + _document->SetError( XML_ERROR_PARSING_CDATA, start, 0 ); + } + return p; + } + else { + int flags = _document->ProcessEntities() ? StrPair::TEXT_ELEMENT : StrPair::TEXT_ELEMENT_LEAVE_ENTITIES; + if ( _document->WhitespaceMode() == COLLAPSE_WHITESPACE ) { + flags |= StrPair::COLLAPSE_WHITESPACE; + } + + p = _value.ParseText( p, "<", flags ); + if ( p && *p ) { + return p-1; + } + if ( !p ) { + _document->SetError( XML_ERROR_PARSING_TEXT, start, 0 ); + } + } + return 0; +} + + +XMLNode* XMLText::ShallowClone( XMLDocument* doc ) const +{ + if ( !doc ) { + doc = _document; + } + XMLText* text = doc->NewText( Value() ); // fixme: this will always allocate memory. Intern? + text->SetCData( this->CData() ); + return text; +} + + +bool XMLText::ShallowEqual( const XMLNode* compare ) const +{ + const XMLText* text = compare->ToText(); + return ( text && XMLUtil::StringEqual( text->Value(), Value() ) ); +} + + +bool XMLText::Accept( XMLVisitor* visitor ) const +{ + TIXMLASSERT( visitor ); + return visitor->Visit( *this ); +} + + +// --------- XMLComment ---------- // + +XMLComment::XMLComment( XMLDocument* doc ) : XMLNode( doc ) +{ +} + + +XMLComment::~XMLComment() +{ +} + + +char* XMLComment::ParseDeep( char* p, StrPair* ) +{ + // Comment parses as text. + const char* start = p; + p = _value.ParseText( p, "-->", StrPair::COMMENT ); + if ( p == 0 ) { + _document->SetError( XML_ERROR_PARSING_COMMENT, start, 0 ); + } + return p; +} + + +XMLNode* XMLComment::ShallowClone( XMLDocument* doc ) const +{ + if ( !doc ) { + doc = _document; + } + XMLComment* comment = doc->NewComment( Value() ); // fixme: this will always allocate memory. Intern? + return comment; +} + + +bool XMLComment::ShallowEqual( const XMLNode* compare ) const +{ + TIXMLASSERT( compare ); + const XMLComment* comment = compare->ToComment(); + return ( comment && XMLUtil::StringEqual( comment->Value(), Value() )); +} + + +bool XMLComment::Accept( XMLVisitor* visitor ) const +{ + TIXMLASSERT( visitor ); + return visitor->Visit( *this ); +} + + +// --------- XMLDeclaration ---------- // + +XMLDeclaration::XMLDeclaration( XMLDocument* doc ) : XMLNode( doc ) +{ +} + + +XMLDeclaration::~XMLDeclaration() +{ + //printf( "~XMLDeclaration\n" ); +} + + +char* XMLDeclaration::ParseDeep( char* p, StrPair* ) +{ + // Declaration parses as text. + const char* start = p; + p = _value.ParseText( p, "?>", StrPair::NEEDS_NEWLINE_NORMALIZATION ); + if ( p == 0 ) { + _document->SetError( XML_ERROR_PARSING_DECLARATION, start, 0 ); + } + return p; +} + + +XMLNode* XMLDeclaration::ShallowClone( XMLDocument* doc ) const +{ + if ( !doc ) { + doc = _document; + } + XMLDeclaration* dec = doc->NewDeclaration( Value() ); // fixme: this will always allocate memory. Intern? + return dec; +} + + +bool XMLDeclaration::ShallowEqual( const XMLNode* compare ) const +{ + TIXMLASSERT( compare ); + const XMLDeclaration* declaration = compare->ToDeclaration(); + return ( declaration && XMLUtil::StringEqual( declaration->Value(), Value() )); +} + + + +bool XMLDeclaration::Accept( XMLVisitor* visitor ) const +{ + TIXMLASSERT( visitor ); + return visitor->Visit( *this ); +} + +// --------- XMLUnknown ---------- // + +XMLUnknown::XMLUnknown( XMLDocument* doc ) : XMLNode( doc ) +{ +} + + +XMLUnknown::~XMLUnknown() +{ +} + + +char* XMLUnknown::ParseDeep( char* p, StrPair* ) +{ + // Unknown parses as text. + const char* start = p; + + p = _value.ParseText( p, ">", StrPair::NEEDS_NEWLINE_NORMALIZATION ); + if ( !p ) { + _document->SetError( XML_ERROR_PARSING_UNKNOWN, start, 0 ); + } + return p; +} + + +XMLNode* XMLUnknown::ShallowClone( XMLDocument* doc ) const +{ + if ( !doc ) { + doc = _document; + } + XMLUnknown* text = doc->NewUnknown( Value() ); // fixme: this will always allocate memory. Intern? + return text; +} + + +bool XMLUnknown::ShallowEqual( const XMLNode* compare ) const +{ + TIXMLASSERT( compare ); + const XMLUnknown* unknown = compare->ToUnknown(); + return ( unknown && XMLUtil::StringEqual( unknown->Value(), Value() )); +} + + +bool XMLUnknown::Accept( XMLVisitor* visitor ) const +{ + TIXMLASSERT( visitor ); + return visitor->Visit( *this ); +} + +// --------- XMLAttribute ---------- // + +const char* XMLAttribute::Name() const +{ + return _name.GetStr(); +} + +const char* XMLAttribute::Value() const +{ + return _value.GetStr(); +} + +char* XMLAttribute::ParseDeep( char* p, bool processEntities ) +{ + // Parse using the name rules: bug fix, was using ParseText before + p = _name.ParseName( p ); + if ( !p || !*p ) { + return 0; + } + + // Skip white space before = + p = XMLUtil::SkipWhiteSpace( p ); + if ( *p != '=' ) { + return 0; + } + + ++p; // move up to opening quote + p = XMLUtil::SkipWhiteSpace( p ); + if ( *p != '\"' && *p != '\'' ) { + return 0; + } + + char endTag[2] = { *p, 0 }; + ++p; // move past opening quote + + p = _value.ParseText( p, endTag, processEntities ? StrPair::ATTRIBUTE_VALUE : StrPair::ATTRIBUTE_VALUE_LEAVE_ENTITIES ); + return p; +} + + +void XMLAttribute::SetName( const char* n ) +{ + _name.SetStr( n ); +} + + +XMLError XMLAttribute::QueryIntValue( int* value ) const +{ + if ( XMLUtil::ToInt( Value(), value )) { + return XML_NO_ERROR; + } + return XML_WRONG_ATTRIBUTE_TYPE; +} + + +XMLError XMLAttribute::QueryUnsignedValue( unsigned int* value ) const +{ + if ( XMLUtil::ToUnsigned( Value(), value )) { + return XML_NO_ERROR; + } + return XML_WRONG_ATTRIBUTE_TYPE; +} + + +XMLError XMLAttribute::QueryBoolValue( bool* value ) const +{ + if ( XMLUtil::ToBool( Value(), value )) { + return XML_NO_ERROR; + } + return XML_WRONG_ATTRIBUTE_TYPE; +} + + +XMLError XMLAttribute::QueryFloatValue( float* value ) const +{ + if ( XMLUtil::ToFloat( Value(), value )) { + return XML_NO_ERROR; + } + return XML_WRONG_ATTRIBUTE_TYPE; +} + + +XMLError XMLAttribute::QueryDoubleValue( double* value ) const +{ + if ( XMLUtil::ToDouble( Value(), value )) { + return XML_NO_ERROR; + } + return XML_WRONG_ATTRIBUTE_TYPE; +} + + +void XMLAttribute::SetAttribute( const char* v ) +{ + _value.SetStr( v ); +} + + +void XMLAttribute::SetAttribute( int v ) +{ + char buf[BUF_SIZE]; + XMLUtil::ToStr( v, buf, BUF_SIZE ); + _value.SetStr( buf ); +} + + +void XMLAttribute::SetAttribute( unsigned v ) +{ + char buf[BUF_SIZE]; + XMLUtil::ToStr( v, buf, BUF_SIZE ); + _value.SetStr( buf ); +} + + +void XMLAttribute::SetAttribute( bool v ) +{ + char buf[BUF_SIZE]; + XMLUtil::ToStr( v, buf, BUF_SIZE ); + _value.SetStr( buf ); +} + +void XMLAttribute::SetAttribute( double v ) +{ + char buf[BUF_SIZE]; + XMLUtil::ToStr( v, buf, BUF_SIZE ); + _value.SetStr( buf ); +} + +void XMLAttribute::SetAttribute( float v ) +{ + char buf[BUF_SIZE]; + XMLUtil::ToStr( v, buf, BUF_SIZE ); + _value.SetStr( buf ); +} + + +// --------- XMLElement ---------- // +XMLElement::XMLElement( XMLDocument* doc ) : XMLNode( doc ), + _closingType( 0 ), + _rootAttribute( 0 ) +{ +} + + +XMLElement::~XMLElement() +{ + while( _rootAttribute ) { + XMLAttribute* next = _rootAttribute->_next; + DeleteAttribute( _rootAttribute ); + _rootAttribute = next; + } +} + + +const XMLAttribute* XMLElement::FindAttribute( const char* name ) const +{ + for( XMLAttribute* a = _rootAttribute; a; a = a->_next ) { + if ( XMLUtil::StringEqual( a->Name(), name ) ) { + return a; + } + } + return 0; +} + + +const char* XMLElement::Attribute( const char* name, const char* value ) const +{ + const XMLAttribute* a = FindAttribute( name ); + if ( !a ) { + return 0; + } + if ( !value || XMLUtil::StringEqual( a->Value(), value )) { + return a->Value(); + } + return 0; +} + + +const char* XMLElement::GetText() const +{ + if ( FirstChild() && FirstChild()->ToText() ) { + return FirstChild()->Value(); + } + return 0; +} + + +void XMLElement::SetText( const char* inText ) +{ + if ( FirstChild() && FirstChild()->ToText() ) + FirstChild()->SetValue( inText ); + else { + XMLText* theText = GetDocument()->NewText( inText ); + InsertFirstChild( theText ); + } +} + + +void XMLElement::SetText( int v ) +{ + char buf[BUF_SIZE]; + XMLUtil::ToStr( v, buf, BUF_SIZE ); + SetText( buf ); +} + + +void XMLElement::SetText( unsigned v ) +{ + char buf[BUF_SIZE]; + XMLUtil::ToStr( v, buf, BUF_SIZE ); + SetText( buf ); +} + + +void XMLElement::SetText( bool v ) +{ + char buf[BUF_SIZE]; + XMLUtil::ToStr( v, buf, BUF_SIZE ); + SetText( buf ); +} + + +void XMLElement::SetText( float v ) +{ + char buf[BUF_SIZE]; + XMLUtil::ToStr( v, buf, BUF_SIZE ); + SetText( buf ); +} + + +void XMLElement::SetText( double v ) +{ + char buf[BUF_SIZE]; + XMLUtil::ToStr( v, buf, BUF_SIZE ); + SetText( buf ); +} + + +XMLError XMLElement::QueryIntText( int* ival ) const +{ + if ( FirstChild() && FirstChild()->ToText() ) { + const char* t = FirstChild()->Value(); + if ( XMLUtil::ToInt( t, ival ) ) { + return XML_SUCCESS; + } + return XML_CAN_NOT_CONVERT_TEXT; + } + return XML_NO_TEXT_NODE; +} + + +XMLError XMLElement::QueryUnsignedText( unsigned* uval ) const +{ + if ( FirstChild() && FirstChild()->ToText() ) { + const char* t = FirstChild()->Value(); + if ( XMLUtil::ToUnsigned( t, uval ) ) { + return XML_SUCCESS; + } + return XML_CAN_NOT_CONVERT_TEXT; + } + return XML_NO_TEXT_NODE; +} + + +XMLError XMLElement::QueryBoolText( bool* bval ) const +{ + if ( FirstChild() && FirstChild()->ToText() ) { + const char* t = FirstChild()->Value(); + if ( XMLUtil::ToBool( t, bval ) ) { + return XML_SUCCESS; + } + return XML_CAN_NOT_CONVERT_TEXT; + } + return XML_NO_TEXT_NODE; +} + + +XMLError XMLElement::QueryDoubleText( double* dval ) const +{ + if ( FirstChild() && FirstChild()->ToText() ) { + const char* t = FirstChild()->Value(); + if ( XMLUtil::ToDouble( t, dval ) ) { + return XML_SUCCESS; + } + return XML_CAN_NOT_CONVERT_TEXT; + } + return XML_NO_TEXT_NODE; +} + + +XMLError XMLElement::QueryFloatText( float* fval ) const +{ + if ( FirstChild() && FirstChild()->ToText() ) { + const char* t = FirstChild()->Value(); + if ( XMLUtil::ToFloat( t, fval ) ) { + return XML_SUCCESS; + } + return XML_CAN_NOT_CONVERT_TEXT; + } + return XML_NO_TEXT_NODE; +} + + + +XMLAttribute* XMLElement::FindOrCreateAttribute( const char* name ) +{ + XMLAttribute* last = 0; + XMLAttribute* attrib = 0; + for( attrib = _rootAttribute; + attrib; + last = attrib, attrib = attrib->_next ) { + if ( XMLUtil::StringEqual( attrib->Name(), name ) ) { + break; + } + } + if ( !attrib ) { + TIXMLASSERT( sizeof( XMLAttribute ) == _document->_attributePool.ItemSize() ); + attrib = new (_document->_attributePool.Alloc() ) XMLAttribute(); + attrib->_memPool = &_document->_attributePool; + if ( last ) { + last->_next = attrib; + } + else { + _rootAttribute = attrib; + } + attrib->SetName( name ); + attrib->_memPool->SetTracked(); // always created and linked. + } + return attrib; +} + + +void XMLElement::DeleteAttribute( const char* name ) +{ + XMLAttribute* prev = 0; + for( XMLAttribute* a=_rootAttribute; a; a=a->_next ) { + if ( XMLUtil::StringEqual( name, a->Name() ) ) { + if ( prev ) { + prev->_next = a->_next; + } + else { + _rootAttribute = a->_next; + } + DeleteAttribute( a ); + break; + } + prev = a; + } +} + + +char* XMLElement::ParseAttributes( char* p ) +{ + const char* start = p; + XMLAttribute* prevAttribute = 0; + + // Read the attributes. + while( p ) { + p = XMLUtil::SkipWhiteSpace( p ); + if ( !(*p) ) { + _document->SetError( XML_ERROR_PARSING_ELEMENT, start, Name() ); + return 0; + } + + // attribute. + if (XMLUtil::IsNameStartChar( *p ) ) { + TIXMLASSERT( sizeof( XMLAttribute ) == _document->_attributePool.ItemSize() ); + XMLAttribute* attrib = new (_document->_attributePool.Alloc() ) XMLAttribute(); + attrib->_memPool = &_document->_attributePool; + attrib->_memPool->SetTracked(); + + p = attrib->ParseDeep( p, _document->ProcessEntities() ); + if ( !p || Attribute( attrib->Name() ) ) { + DeleteAttribute( attrib ); + _document->SetError( XML_ERROR_PARSING_ATTRIBUTE, start, p ); + return 0; + } + // There is a minor bug here: if the attribute in the source xml + // document is duplicated, it will not be detected and the + // attribute will be doubly added. However, tracking the 'prevAttribute' + // avoids re-scanning the attribute list. Preferring performance for + // now, may reconsider in the future. + if ( prevAttribute ) { + prevAttribute->_next = attrib; + } + else { + _rootAttribute = attrib; + } + prevAttribute = attrib; + } + // end of the tag + else if ( *p == '/' && *(p+1) == '>' ) { + _closingType = CLOSED; + return p+2; // done; sealed element. + } + // end of the tag + else if ( *p == '>' ) { + ++p; + break; + } + else { + _document->SetError( XML_ERROR_PARSING_ELEMENT, start, p ); + return 0; + } + } + return p; +} + +void XMLElement::DeleteAttribute( XMLAttribute* attribute ) +{ + if ( attribute == 0 ) { + return; + } + MemPool* pool = attribute->_memPool; + attribute->~XMLAttribute(); + pool->Free( attribute ); +} + +// +// +// foobar +// +char* XMLElement::ParseDeep( char* p, StrPair* strPair ) +{ + // Read the element name. + p = XMLUtil::SkipWhiteSpace( p ); + + // The closing element is the form. It is + // parsed just like a regular element then deleted from + // the DOM. + if ( *p == '/' ) { + _closingType = CLOSING; + ++p; + } + + p = _value.ParseName( p ); + if ( _value.Empty() ) { + return 0; + } + + p = ParseAttributes( p ); + if ( !p || !*p || _closingType ) { + return p; + } + + p = XMLNode::ParseDeep( p, strPair ); + return p; +} + + + +XMLNode* XMLElement::ShallowClone( XMLDocument* doc ) const +{ + if ( !doc ) { + doc = _document; + } + XMLElement* element = doc->NewElement( Value() ); // fixme: this will always allocate memory. Intern? + for( const XMLAttribute* a=FirstAttribute(); a; a=a->Next() ) { + element->SetAttribute( a->Name(), a->Value() ); // fixme: this will always allocate memory. Intern? + } + return element; +} + + +bool XMLElement::ShallowEqual( const XMLNode* compare ) const +{ + TIXMLASSERT( compare ); + const XMLElement* other = compare->ToElement(); + if ( other && XMLUtil::StringEqual( other->Value(), Value() )) { + + const XMLAttribute* a=FirstAttribute(); + const XMLAttribute* b=other->FirstAttribute(); + + while ( a && b ) { + if ( !XMLUtil::StringEqual( a->Value(), b->Value() ) ) { + return false; + } + a = a->Next(); + b = b->Next(); + } + if ( a || b ) { + // different count + return false; + } + return true; + } + return false; +} + + +bool XMLElement::Accept( XMLVisitor* visitor ) const +{ + TIXMLASSERT( visitor ); + if ( visitor->VisitEnter( *this, _rootAttribute ) ) { + for ( const XMLNode* node=FirstChild(); node; node=node->NextSibling() ) { + if ( !node->Accept( visitor ) ) { + break; + } + } + } + return visitor->VisitExit( *this ); +} + + +// --------- XMLDocument ----------- // + +// Warning: List must match 'enum XMLError' +const char* XMLDocument::_errorNames[XML_ERROR_COUNT] = { + "XML_SUCCESS", + "XML_NO_ATTRIBUTE", + "XML_WRONG_ATTRIBUTE_TYPE", + "XML_ERROR_FILE_NOT_FOUND", + "XML_ERROR_FILE_COULD_NOT_BE_OPENED", + "XML_ERROR_FILE_READ_ERROR", + "XML_ERROR_ELEMENT_MISMATCH", + "XML_ERROR_PARSING_ELEMENT", + "XML_ERROR_PARSING_ATTRIBUTE", + "XML_ERROR_IDENTIFYING_TAG", + "XML_ERROR_PARSING_TEXT", + "XML_ERROR_PARSING_CDATA", + "XML_ERROR_PARSING_COMMENT", + "XML_ERROR_PARSING_DECLARATION", + "XML_ERROR_PARSING_UNKNOWN", + "XML_ERROR_EMPTY_DOCUMENT", + "XML_ERROR_MISMATCHED_ELEMENT", + "XML_ERROR_PARSING", + "XML_CAN_NOT_CONVERT_TEXT", + "XML_NO_TEXT_NODE" +}; + + +XMLDocument::XMLDocument( bool processEntities, Whitespace whitespace ) : + XMLNode( 0 ), + _writeBOM( false ), + _processEntities( processEntities ), + _errorID( XML_NO_ERROR ), + _whitespace( whitespace ), + _errorStr1( 0 ), + _errorStr2( 0 ), + _charBuffer( 0 ) +{ + _document = this; // avoid warning about 'this' in initializer list +} + + +XMLDocument::~XMLDocument() +{ + Clear(); +} + + +void XMLDocument::Clear() +{ + DeleteChildren(); + +#ifdef DEBUG + const bool hadError = Error(); +#endif + _errorID = XML_NO_ERROR; + _errorStr1 = 0; + _errorStr2 = 0; + + delete [] _charBuffer; + _charBuffer = 0; + +#if 0 + _textPool.Trace( "text" ); + _elementPool.Trace( "element" ); + _commentPool.Trace( "comment" ); + _attributePool.Trace( "attribute" ); +#endif + +#ifdef DEBUG + if ( !hadError ) { + TIXMLASSERT( _elementPool.CurrentAllocs() == _elementPool.Untracked() ); + TIXMLASSERT( _attributePool.CurrentAllocs() == _attributePool.Untracked() ); + TIXMLASSERT( _textPool.CurrentAllocs() == _textPool.Untracked() ); + TIXMLASSERT( _commentPool.CurrentAllocs() == _commentPool.Untracked() ); + } +#endif +} + + +XMLElement* XMLDocument::NewElement( const char* name ) +{ + TIXMLASSERT( sizeof( XMLElement ) == _elementPool.ItemSize() ); + XMLElement* ele = new (_elementPool.Alloc()) XMLElement( this ); + ele->_memPool = &_elementPool; + ele->SetName( name ); + return ele; +} + + +XMLComment* XMLDocument::NewComment( const char* str ) +{ + TIXMLASSERT( sizeof( XMLComment ) == _commentPool.ItemSize() ); + XMLComment* comment = new (_commentPool.Alloc()) XMLComment( this ); + comment->_memPool = &_commentPool; + comment->SetValue( str ); + return comment; +} + + +XMLText* XMLDocument::NewText( const char* str ) +{ + TIXMLASSERT( sizeof( XMLText ) == _textPool.ItemSize() ); + XMLText* text = new (_textPool.Alloc()) XMLText( this ); + text->_memPool = &_textPool; + text->SetValue( str ); + return text; +} + + +XMLDeclaration* XMLDocument::NewDeclaration( const char* str ) +{ + TIXMLASSERT( sizeof( XMLDeclaration ) == _commentPool.ItemSize() ); + XMLDeclaration* dec = new (_commentPool.Alloc()) XMLDeclaration( this ); + dec->_memPool = &_commentPool; + dec->SetValue( str ? str : "xml version=\"1.0\" encoding=\"UTF-8\"" ); + return dec; +} + + +XMLUnknown* XMLDocument::NewUnknown( const char* str ) +{ + TIXMLASSERT( sizeof( XMLUnknown ) == _commentPool.ItemSize() ); + XMLUnknown* unk = new (_commentPool.Alloc()) XMLUnknown( this ); + unk->_memPool = &_commentPool; + unk->SetValue( str ); + return unk; +} + +static FILE* callfopen( const char* filepath, const char* mode ) +{ + TIXMLASSERT( filepath ); + TIXMLASSERT( mode ); +#if defined(_MSC_VER) && (_MSC_VER >= 1400 ) && (!defined WINCE) + FILE* fp = 0; + errno_t err = fopen_s( &fp, filepath, mode ); + if ( err ) { + return 0; + } +#else + FILE* fp = fopen( filepath, mode ); +#endif + return fp; +} + +void XMLDocument::DeleteNode( XMLNode* node ) { + TIXMLASSERT( node ); + TIXMLASSERT(node->_document == this ); + if (node->_parent) { + node->_parent->DeleteChild( node ); + } + else { + // Isn't in the tree. + // Use the parent delete. + // Also, we need to mark it tracked: we 'know' + // it was never used. + node->_memPool->SetTracked(); + // Call the static XMLNode version: + XMLNode::DeleteNode(node); + } +} + + +XMLError XMLDocument::LoadFile( const char* filename ) +{ + Clear(); + FILE* fp = callfopen( filename, "rb" ); + if ( !fp ) { + SetError( XML_ERROR_FILE_NOT_FOUND, filename, 0 ); + return _errorID; + } + LoadFile( fp ); + fclose( fp ); + return _errorID; +} + + +XMLError XMLDocument::LoadFile( FILE* fp ) +{ + Clear(); + + fseek( fp, 0, SEEK_SET ); + if ( fgetc( fp ) == EOF && ferror( fp ) != 0 ) { + SetError( XML_ERROR_FILE_READ_ERROR, 0, 0 ); + return _errorID; + } + + fseek( fp, 0, SEEK_END ); + const long filelength = ftell( fp ); + fseek( fp, 0, SEEK_SET ); + if ( filelength == -1L ) { + SetError( XML_ERROR_FILE_READ_ERROR, 0, 0 ); + return _errorID; + } + + const size_t size = filelength; + if ( size == 0 ) { + SetError( XML_ERROR_EMPTY_DOCUMENT, 0, 0 ); + return _errorID; + } + + _charBuffer = new char[size+1]; + size_t read = fread( _charBuffer, 1, size, fp ); + if ( read != size ) { + SetError( XML_ERROR_FILE_READ_ERROR, 0, 0 ); + return _errorID; + } + + _charBuffer[size] = 0; + + Parse(); + return _errorID; +} + + +XMLError XMLDocument::SaveFile( const char* filename, bool compact ) +{ + FILE* fp = callfopen( filename, "w" ); + if ( !fp ) { + SetError( XML_ERROR_FILE_COULD_NOT_BE_OPENED, filename, 0 ); + return _errorID; + } + SaveFile(fp, compact); + fclose( fp ); + return _errorID; +} + + +XMLError XMLDocument::SaveFile( FILE* fp, bool compact ) +{ + XMLPrinter stream( fp, compact ); + Print( &stream ); + return _errorID; +} + + +XMLError XMLDocument::Parse( const char* p, size_t len ) +{ + Clear(); + + if ( len == 0 || !p || !*p ) { + SetError( XML_ERROR_EMPTY_DOCUMENT, 0, 0 ); + return _errorID; + } + if ( len == (size_t)(-1) ) { + len = strlen( p ); + } + _charBuffer = new char[ len+1 ]; + memcpy( _charBuffer, p, len ); + _charBuffer[len] = 0; + + Parse(); + if ( Error() ) { + // clean up now essentially dangling memory. + // and the parse fail can put objects in the + // pools that are dead and inaccessible. + DeleteChildren(); + _elementPool.Clear(); + _attributePool.Clear(); + _textPool.Clear(); + _commentPool.Clear(); + } + return _errorID; +} + + +void XMLDocument::Print( XMLPrinter* streamer ) const +{ + XMLPrinter stdStreamer( stdout ); + if ( !streamer ) { + streamer = &stdStreamer; + } + Accept( streamer ); +} + + +void XMLDocument::SetError( XMLError error, const char* str1, const char* str2 ) +{ + TIXMLASSERT( error >= 0 && error < XML_ERROR_COUNT ); + _errorID = error; + _errorStr1 = str1; + _errorStr2 = str2; +} + +const char* XMLDocument::ErrorName() const +{ + TIXMLASSERT( _errorID >= 0 && _errorID < XML_ERROR_COUNT ); + return _errorNames[_errorID]; +} + +void XMLDocument::PrintError() const +{ + if ( Error() ) { + static const int LEN = 20; + char buf1[LEN] = { 0 }; + char buf2[LEN] = { 0 }; + + if ( _errorStr1 ) { + TIXML_SNPRINTF( buf1, LEN, "%s", _errorStr1 ); + } + if ( _errorStr2 ) { + TIXML_SNPRINTF( buf2, LEN, "%s", _errorStr2 ); + } + + printf( "XMLDocument error id=%d '%s' str1=%s str2=%s\n", + _errorID, ErrorName(), buf1, buf2 ); + } +} + +void XMLDocument::Parse() +{ + TIXMLASSERT( NoChildren() ); // Clear() must have been called previously + TIXMLASSERT( _charBuffer ); + char* p = _charBuffer; + p = XMLUtil::SkipWhiteSpace( p ); + p = const_cast( XMLUtil::ReadBOM( p, &_writeBOM ) ); + if ( !*p ) { + SetError( XML_ERROR_EMPTY_DOCUMENT, 0, 0 ); + return; + } + ParseDeep(p, 0 ); +} + +XMLPrinter::XMLPrinter( FILE* file, bool compact, int depth ) : + _elementJustOpened( false ), + _firstElement( true ), + _fp( file ), + _depth( depth ), + _textDepth( -1 ), + _processEntities( true ), + _compactMode( compact ) +{ + for( int i=0; i'] = true; // not required, but consistency is nice + _buffer.Push( 0 ); +} + + +void XMLPrinter::Print( const char* format, ... ) +{ + va_list va; + va_start( va, format ); + + if ( _fp ) { + vfprintf( _fp, format, va ); + } + else { +#if defined(_MSC_VER) && (_MSC_VER >= 1400 ) + #if defined(WINCE) + int len = 512; + do { + len = len*2; + char* str = new char[len](); + len = _vsnprintf(str, len, format, va); + delete[] str; + }while (len < 0); + #else + int len = _vscprintf( format, va ); + #endif +#else + int len = vsnprintf( 0, 0, format, va ); +#endif + // Close out and re-start the va-args + va_end( va ); + va_start( va, format ); + TIXMLASSERT( _buffer.Size() > 0 && _buffer[_buffer.Size() - 1] == 0 ); + char* p = _buffer.PushArr( len ) - 1; // back up over the null terminator. +#if defined(_MSC_VER) && (_MSC_VER >= 1400 ) + #if defined(WINCE) + _vsnprintf( p, len+1, format, va ); + #else + vsnprintf_s( p, len+1, _TRUNCATE, format, va ); + #endif +#else + vsnprintf( p, len+1, format, va ); +#endif + } + va_end( va ); +} + + +void XMLPrinter::PrintSpace( int depth ) +{ + for( int i=0; i 0 && *q < ENTITY_RANGE ) { + // Check for entities. If one is found, flush + // the stream up until the entity, write the + // entity, and keep looking. + if ( flag[(unsigned char)(*q)] ) { + while ( p < q ) { + Print( "%c", *p ); + ++p; + } + for( int i=0; i 0) ) { + Print( "%s", p ); + } +} + + +void XMLPrinter::PushHeader( bool writeBOM, bool writeDec ) +{ + if ( writeBOM ) { + static const unsigned char bom[] = { TIXML_UTF_LEAD_0, TIXML_UTF_LEAD_1, TIXML_UTF_LEAD_2, 0 }; + Print( "%s", bom ); + } + if ( writeDec ) { + PushDeclaration( "xml version=\"1.0\"" ); + } +} + + +void XMLPrinter::OpenElement( const char* name, bool compactMode ) +{ + SealElementIfJustOpened(); + _stack.Push( name ); + + if ( _textDepth < 0 && !_firstElement && !compactMode ) { + Print( "\n" ); + } + if ( !compactMode ) { + PrintSpace( _depth ); + } + + Print( "<%s", name ); + _elementJustOpened = true; + _firstElement = false; + ++_depth; +} + + +void XMLPrinter::PushAttribute( const char* name, const char* value ) +{ + TIXMLASSERT( _elementJustOpened ); + Print( " %s=\"", name ); + PrintString( value, false ); + Print( "\"" ); +} + + +void XMLPrinter::PushAttribute( const char* name, int v ) +{ + char buf[BUF_SIZE]; + XMLUtil::ToStr( v, buf, BUF_SIZE ); + PushAttribute( name, buf ); +} + + +void XMLPrinter::PushAttribute( const char* name, unsigned v ) +{ + char buf[BUF_SIZE]; + XMLUtil::ToStr( v, buf, BUF_SIZE ); + PushAttribute( name, buf ); +} + + +void XMLPrinter::PushAttribute( const char* name, bool v ) +{ + char buf[BUF_SIZE]; + XMLUtil::ToStr( v, buf, BUF_SIZE ); + PushAttribute( name, buf ); +} + + +void XMLPrinter::PushAttribute( const char* name, double v ) +{ + char buf[BUF_SIZE]; + XMLUtil::ToStr( v, buf, BUF_SIZE ); + PushAttribute( name, buf ); +} + + +void XMLPrinter::CloseElement( bool compactMode ) +{ + --_depth; + const char* name = _stack.Pop(); + + if ( _elementJustOpened ) { + Print( "/>" ); + } + else { + if ( _textDepth < 0 && !compactMode) { + Print( "\n" ); + PrintSpace( _depth ); + } + Print( "", name ); + } + + if ( _textDepth == _depth ) { + _textDepth = -1; + } + if ( _depth == 0 && !compactMode) { + Print( "\n" ); + } + _elementJustOpened = false; +} + + +void XMLPrinter::SealElementIfJustOpened() +{ + if ( !_elementJustOpened ) { + return; + } + _elementJustOpened = false; + Print( ">" ); +} + + +void XMLPrinter::PushText( const char* text, bool cdata ) +{ + _textDepth = _depth-1; + + SealElementIfJustOpened(); + if ( cdata ) { + Print( "" ); + } + else { + PrintString( text, true ); + } +} + +void XMLPrinter::PushText( int value ) +{ + char buf[BUF_SIZE]; + XMLUtil::ToStr( value, buf, BUF_SIZE ); + PushText( buf, false ); +} + + +void XMLPrinter::PushText( unsigned value ) +{ + char buf[BUF_SIZE]; + XMLUtil::ToStr( value, buf, BUF_SIZE ); + PushText( buf, false ); +} + + +void XMLPrinter::PushText( bool value ) +{ + char buf[BUF_SIZE]; + XMLUtil::ToStr( value, buf, BUF_SIZE ); + PushText( buf, false ); +} + + +void XMLPrinter::PushText( float value ) +{ + char buf[BUF_SIZE]; + XMLUtil::ToStr( value, buf, BUF_SIZE ); + PushText( buf, false ); +} + + +void XMLPrinter::PushText( double value ) +{ + char buf[BUF_SIZE]; + XMLUtil::ToStr( value, buf, BUF_SIZE ); + PushText( buf, false ); +} + + +void XMLPrinter::PushComment( const char* comment ) +{ + SealElementIfJustOpened(); + if ( _textDepth < 0 && !_firstElement && !_compactMode) { + Print( "\n" ); + PrintSpace( _depth ); + } + _firstElement = false; + Print( "", comment ); +} + + +void XMLPrinter::PushDeclaration( const char* value ) +{ + SealElementIfJustOpened(); + if ( _textDepth < 0 && !_firstElement && !_compactMode) { + Print( "\n" ); + PrintSpace( _depth ); + } + _firstElement = false; + Print( "", value ); +} + + +void XMLPrinter::PushUnknown( const char* value ) +{ + SealElementIfJustOpened(); + if ( _textDepth < 0 && !_firstElement && !_compactMode) { + Print( "\n" ); + PrintSpace( _depth ); + } + _firstElement = false; + Print( "", value ); +} + + +bool XMLPrinter::VisitEnter( const XMLDocument& doc ) +{ + _processEntities = doc.ProcessEntities(); + if ( doc.HasBOM() ) { + PushHeader( true, false ); + } + return true; +} + + +bool XMLPrinter::VisitEnter( const XMLElement& element, const XMLAttribute* attribute ) +{ + const XMLElement* parentElem = element.Parent()->ToElement(); + bool compactMode = parentElem ? CompactMode(*parentElem) : _compactMode; + OpenElement( element.Name(), compactMode ); + while ( attribute ) { + PushAttribute( attribute->Name(), attribute->Value() ); + attribute = attribute->Next(); + } + return true; +} + + +bool XMLPrinter::VisitExit( const XMLElement& element ) +{ + CloseElement( CompactMode(element) ); + return true; +} + + +bool XMLPrinter::Visit( const XMLText& text ) +{ + PushText( text.Value(), text.CData() ); + return true; +} + + +bool XMLPrinter::Visit( const XMLComment& comment ) +{ + PushComment( comment.Value() ); + return true; +} + +bool XMLPrinter::Visit( const XMLDeclaration& declaration ) +{ + PushDeclaration( declaration.Value() ); + return true; +} + + +bool XMLPrinter::Visit( const XMLUnknown& unknown ) +{ + PushUnknown( unknown.Value() ); + return true; +} + +} // namespace tinyxml2 + diff --git a/src/tinyxml2_parser.cpp b/src/tinyxml2_parser.cpp new file mode 100644 index 0000000..11a4658 --- /dev/null +++ b/src/tinyxml2_parser.cpp @@ -0,0 +1,265 @@ +#include "tinyxml2_parser.h" + +#include "utility.h" + +namespace azure { namespace storage_lite { + +std::string tinyxml2_parser::parse_text(tinyxml2::XMLElement *ele, const std::string &name) const +{ + std::string text; + ele = ele->FirstChildElement(name.data()); + if (ele && ele->FirstChild()) { + text = ele->FirstChild()->ToText()->Value(); + } + + return text; +} + +unsigned long long tinyxml2_parser::parse_long(tinyxml2::XMLElement *ele, const std::string &name) const +{ + unsigned long long result = 0; + + std::string text = parse_text(ele, name); + if (!text.empty()) { + std::istringstream iss(text); + iss >> result; + } + + return result; +} + +storage_error tinyxml2_parser::parse_storage_error(const std::string &xml) const +{ + storage_error error; + + tinyxml2::XMLDocument xdoc; + if (xdoc.Parse(xml.data(), xml.size()) == tinyxml2::XMLError::XML_SUCCESS) + { + auto xerror = xdoc.FirstChildElement("Error"); + error.code_name = parse_text(xerror, "Code"); + error.message = parse_text(xerror, "Message"); + } + + return error; +} + +list_containers_item tinyxml2_parser::parse_list_containers_item(tinyxml2::XMLElement *ele) const +{ + list_containers_item item; + + item.name = parse_text(ele, "Name"); + + auto xproperty = ele->FirstChildElement("Properties"); + item.etag = parse_text(xproperty, "Etag"); + item.last_modified = parse_text(xproperty, "Last-Modified"); + item.status = parse_lease_status(parse_text(xproperty, "LeaseStatus")); + item.state = parse_lease_state(parse_text(xproperty, "LeaseState")); + item.duration = parse_lease_duration(parse_text(xproperty, "LeaseDuration")); + + //TODO: parse_metadata + + return item; +} + +list_constainers_segmented_response tinyxml2_parser::parse_list_constainers_segmented_response(const std::string &xml) const +{ + list_constainers_segmented_response response; + + tinyxml2::XMLDocument xdoc; + if (xdoc.Parse(xml.data(), xml.size()) == tinyxml2::XML_SUCCESS) + { + auto xresults = xdoc.FirstChildElement("EnumerationResults"); + response.next_marker = parse_text(xresults, "NextMarker"); + auto xitems = xresults->FirstChildElement("Containers"); + auto xitem = xitems->FirstChildElement("Container"); + while (xitem) { + response.containers.push_back(parse_list_containers_item(xitem)); + xitem = xitem->NextSiblingElement("Container"); + } + } + + return response; +} + +list_blobs_item tinyxml2_parser::parse_list_blobs_item(tinyxml2::XMLElement *ele) const +{ + list_blobs_item item; + + item.name = parse_text(ele, "Name"); + + auto xproperty = ele->FirstChildElement("Properties"); + item.etag = parse_text(xproperty, "Etag"); + item.last_modified = parse_text(xproperty, "Last-Modified"); + item.cache_control = parse_text(xproperty, "Cache-Control"); + item.content_encoding = parse_text(xproperty, "Content-Encoding"); + item.content_language = parse_text(xproperty, "Content-Language"); + item.content_type = parse_text(xproperty, "Content-Type"); + item.content_md5 = parse_text(xproperty, "Content-MD5"); + item.content_length = parse_long(xproperty, "Content-Length"); + item.status = parse_lease_status(parse_text(xproperty, "LeaseStatus")); + item.state = parse_lease_state(parse_text(xproperty, "LeaseState")); + item.duration = parse_lease_duration(parse_text(xproperty, "LeaseDuration")); + + //parse_metadata + + return item; +} + +list_blobs_response tinyxml2_parser::parse_list_blobs_response(const std::string &xml) const +{ + list_blobs_response response; + + tinyxml2::XMLDocument xdoc; + if (xdoc.Parse(xml.data(), xml.size()) == tinyxml2::XML_SUCCESS) { + auto xresults = xdoc.FirstChildElement("EnumerationResults"); + response.next_marker = parse_text(xresults, "NextMarker"); + auto xitems = xresults->FirstChildElement("Blobs"); + auto xitem = xitems->FirstChildElement("Blob"); + while (xitem) { + response.blobs.push_back(parse_list_blobs_item(xitem)); + xitem = xitem->NextSiblingElement("Blob"); + } + } + + return response; +} + +std::vector> tinyxml2_parser::parse_blob_metadata(tinyxml2::XMLElement *ele) const +{ + std::vector> metadata; + tinyxml2::XMLElement *current = ele->FirstChildElement(); + while (current) + { + std::string name(current->Name()); + std::string value(current->GetText()); + metadata.push_back(make_pair(name, value)); + current = current->NextSiblingElement(); + } + return metadata; +} + +list_blobs_segmented_item tinyxml2_parser::parse_list_blobs_segmented_item(tinyxml2::XMLElement *ele, bool is_directory) const +{ + list_blobs_segmented_item item; + + item.name = parse_text(ele, "Name"); + item.is_directory = is_directory; + if (!is_directory) + { + auto xproperty = ele->FirstChildElement("Properties"); + item.etag = parse_text(xproperty, "Etag"); + item.last_modified = parse_text(xproperty, "Last-Modified"); + item.cache_control = parse_text(xproperty, "Cache-Control"); + item.content_encoding = parse_text(xproperty, "Content-Encoding"); + item.content_language = parse_text(xproperty, "Content-Language"); + item.content_type = parse_text(xproperty, "Content-Type"); + item.content_md5 = parse_text(xproperty, "Content-MD5"); + item.content_length = parse_long(xproperty, "Content-Length"); + item.status = parse_lease_status(parse_text(xproperty, "LeaseStatus")); + item.state = parse_lease_state(parse_text(xproperty, "LeaseState")); + item.duration = parse_lease_duration(parse_text(xproperty, "LeaseDuration")); + auto xmetadata = ele->FirstChildElement("Metadata"); + if (xmetadata) + { + item.metadata = parse_blob_metadata(xmetadata); + } + } + //parse_metadata + + return item; +} + +list_blobs_segmented_response tinyxml2_parser::parse_list_blobs_segmented_response(const std::string &xml) const +{ + list_blobs_segmented_response response; + + tinyxml2::XMLDocument xdoc; + if (xdoc.Parse(xml.data(), xml.size()) == tinyxml2::XML_SUCCESS) + { + auto xresults = xdoc.FirstChildElement("EnumerationResults"); + response.next_marker = parse_text(xresults, "NextMarker"); + auto xitems = xresults->FirstChildElement("Blobs"); + auto xitem = xitems->FirstChildElement("Blob"); + while (xitem) + { + response.blobs.push_back(parse_list_blobs_segmented_item(xitem, false)); + xitem = xitem->NextSiblingElement("Blob"); + } + + auto xdir = xitems->FirstChildElement("BlobPrefix"); + while (xdir) + { + response.blobs.push_back(parse_list_blobs_segmented_item(xdir, true)); + xdir = xdir->NextSiblingElement("BlobPrefix"); + } + } + + return response; +} + + +get_block_list_item tinyxml2_parser::parse_get_block_list_item(tinyxml2::XMLElement *ele) const +{ + get_block_list_item item; + + item.name = parse_text(ele, "Name"); + item.size = parse_long(ele, "Size"); + + return item; +} + +get_block_list_response tinyxml2_parser::parse_get_block_list_response(const std::string &xml) const +{ + get_block_list_response response; + + tinyxml2::XMLDocument xdoc; + if (xdoc.Parse(xml.data(), xml.size()) == tinyxml2::XML_SUCCESS) + { + auto xresults = xdoc.FirstChildElement("BlockList"); + auto xitems = xresults->FirstChildElement("CommittedBlocks"); + auto xitem = xitems->FirstChildElement("Block"); + while (xitem) { + response.committed.push_back(parse_get_block_list_item(xitem)); + xitem = xitem->NextSiblingElement("Block"); + } + + xitems = xresults->FirstChildElement("UncommittedBlocks"); + xitem = xitems->FirstChildElement("Block"); + while (xitem) { + response.uncommitted.push_back(parse_get_block_list_item(xitem)); + xitem = xitem->NextSiblingElement("Block"); + } + } + + return response; +} + +get_page_ranges_item tinyxml2_parser::parse_get_page_ranges_item(tinyxml2::XMLElement *ele) const +{ + get_page_ranges_item item; + + item.start = parse_long(ele, "Start"); + item.end = parse_long(ele, "End"); + + return item; +} + +get_page_ranges_response tinyxml2_parser::parse_get_page_ranges_response(const std::string &xml) const +{ + get_page_ranges_response response; + + tinyxml2::XMLDocument xdoc; + if (xdoc.Parse(xml.data(), xml.size()) == tinyxml2::XML_SUCCESS) + { + auto xresults = xdoc.FirstChildElement("PageList"); + auto xitem = xresults->FirstChildElement("PageRange"); + while (xitem) { + response.pagelist.push_back(parse_get_page_ranges_item(xitem)); + xitem = xitem->NextSiblingElement("PageRange"); + } + } + + return response; +} + +}} // azure::storage_lite diff --git a/src/utility.cpp b/src/utility.cpp new file mode 100644 index 0000000..ea34c00 --- /dev/null +++ b/src/utility.cpp @@ -0,0 +1,110 @@ +#include + +#include "utility.h" + +#include "constants.h" + +#ifdef WIN32 +#include +#else +#include +#endif + +namespace azure { namespace storage_lite { + + std::string get_uuid() + { + uuid_t uuid; + char uuid_cstr[37]; // 36 byte uuid plus null. + uuid_generate(uuid); + uuid_unparse(uuid, uuid_cstr); + + return std::string(uuid_cstr); + } + + std::string get_ms_date(date_format format) + { + char buf[30]; + std::time_t t = std::time(nullptr); + std::tm *pm; +#ifdef _WIN32 + std::tm m; + pm = &m; + gmtime_s(pm, &t); +#else + pm = std::gmtime(&t); +#endif + size_t s = std::strftime(buf, 30, (format == date_format::iso_8601 ? constants::date_format_iso_8601 : constants::date_format_rfc_1123), pm); + return std::string(buf, s); + } + + std::string get_ms_range(unsigned long long start_byte, unsigned long long end_byte) + { + std::string result("bytes="); + result.append(std::to_string(start_byte)).append("-"); + if (end_byte != 0) { + result.append(std::to_string(end_byte)); + } + return result; + } + + std::string get_http_verb(http_base::http_method method) + { + switch (method) + { + case http_base::http_method::del: + return constants::http_delete; + case http_base::http_method::get: + return constants::http_get; + case http_base::http_method::head: + return constants::http_head; + case http_base::http_method::post: + return constants::http_post; + case http_base::http_method::put: + return constants::http_put; + } + return std::string(); + } + + void add_access_condition_headers(http_base &h, storage_headers &headers, const blob_request_base &r) + { + if (!r.if_modified_since().empty()) + { + h.add_header(constants::header_if_modified_since, r.if_modified_since()); + headers.if_modified_since = r.if_modified_since(); + } + if (!r.if_match().empty()) + { + h.add_header(constants::header_if_match, r.if_match()); + headers.if_match = r.if_match(); + } + if (!r.if_none_match().empty()) + { + h.add_header(constants::header_if_none_match, r.if_none_match()); + headers.if_none_match = r.if_none_match(); + } + if (!r.if_unmodified_since().empty()) + { + h.add_header(constants::header_if_unmodified_since, r.if_unmodified_since()); + headers.if_unmodified_since = r.if_unmodified_since(); + } + } + + bool retryable(http_base::http_code status_code) + { + if (status_code == 408 /*Request Timeout*/) + { + return true; + } + if (status_code >= 300 && status_code < 500) + { + return false; + } + if (status_code == 501 /*Not Implemented*/ || status_code == 505 /*HTTP Version Not Supported*/) + { + return false; + } + return true; + } + +}} // azure::storage_lite diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt new file mode 100644 index 0000000..84528f5 --- /dev/null +++ b/test/CMakeLists.txt @@ -0,0 +1,28 @@ +cmake_minimum_required(VERSION 2.8) + +project(azure_storage_test) + +set(AZURE_STORAGE_TEST_HEADER + catch2/catch.hpp + test_constants.h + test_base.h + integration/blob_integration_base.h +) + +set(AZURE_STORAGE_TEST_SOURCE + test_constants.cpp + test_base.cpp + integration/blob_integration_base.cpp + + integration/blob_container_integration_test.cpp + integration/blob_general_integration_test.cpp + integration/block_blob_integration_test.cpp + integration/append_blob_integration_test.cpp + integration/page_blob_integration_test.cpp +) + + +add_executable(azure-storage-test ${AZURE_STORAGE_TEST_SOURCE}) +target_include_directories(azure-storage-test INTERFACE ${AZURE_STORAGE_TEST_HEADER}) + +target_link_libraries(azure-storage-test azure-storage-lite) \ No newline at end of file diff --git a/test/CMakeLists.txt.in b/test/CMakeLists.txt.in new file mode 100644 index 0000000..a8bcf85 --- /dev/null +++ b/test/CMakeLists.txt.in @@ -0,0 +1,15 @@ +cmake_minimum_required(VERSION 2.8) + +project(googletest-download NONE) + +include(ExternalProject) +ExternalProject_Add(googletest + GIT_REPOSITORY https://github.com/google/googletest.git + GIT_TAG master + SOURCE_DIR "${CMAKE_BINARY_DIR}/googletest-src" + BINARY_DIR "${CMAKE_BINARY_DIR}/googletest-build" + CONFIGURE_COMMAND "" + BUILD_COMMAND "" + INSTALL_COMMAND "" + TEST_COMMAND "" +) \ No newline at end of file diff --git a/test/integration/append_blob_integration_test.cpp b/test/integration/append_blob_integration_test.cpp new file mode 100644 index 0000000..7497bab --- /dev/null +++ b/test/integration/append_blob_integration_test.cpp @@ -0,0 +1,92 @@ +#include "blob_integration_base.h" + +#include "../catch2/catch.hpp" + +TEST_CASE("Create append blob", "[append blob],[blob_service]") +{ + azure::storage_lite::blob_client client = as_test::base::test_blob_client(); + std::string container_name = as_test::create_random_container("", client); + std::string blob_name = as_test::get_random_string(20); + + SECTION("Create append blob with valid name successfully") + { + auto create_append_blob_outcome = client.create_append_blob(container_name, blob_name).get(); + REQUIRE(create_append_blob_outcome.success()); + } + + SECTION("Create append blob with invalid container unsuccessfully") + { + auto create_append_blob_outcome = client.create_append_blob(as_test::get_random_string(20), blob_name).get(); + REQUIRE(!create_append_blob_outcome.success()); + } + + client.delete_container(container_name); +} + +TEST_CASE("Append block from stream", "[append blob],[blob_service]") +{ + azure::storage_lite::blob_client client = as_test::base::test_blob_client(); + std::string container_name = as_test::create_random_container("", client); + std::string blob_name = as_test::get_random_string(20); + auto create_append_blob_outcome = client.create_append_blob(container_name, blob_name).get(); + REQUIRE(create_append_blob_outcome.success()); + + SECTION("Append 4MB block successfully") + { + auto iss = as_test::get_istringstream_with_random_buffer(4 * 1024 * 1024); + auto append_block_from_stream_outcome = client.append_block_from_stream(container_name, blob_name, iss).get(); + REQUIRE(append_block_from_stream_outcome.success()); + + auto get_blob_property_outcome = client.get_blob_property(container_name, blob_name); + REQUIRE(get_blob_property_outcome.success()); + REQUIRE(get_blob_property_outcome.response().size == 4 * 1024 * 1024); + + std::stringbuf strbuf; + std::ostream os(&strbuf); + auto get_blob_outcome = client.download_blob_to_stream(container_name, blob_name, 0, 4 * 1024 * 1024, os).get(); + REQUIRE(get_blob_outcome.success()); + REQUIRE(strbuf.str() == iss.str()); + } + + SECTION("Append 4MB block 10 times successfully") + { + std::string blob_content; + for (unsigned i = 0; i < 10; ++i) + { + auto iss = as_test::get_istringstream_with_random_buffer(4 * 1024 * 1024); + auto append_block_from_stream_outcome = client.append_block_from_stream(container_name, blob_name, iss).get(); + REQUIRE(append_block_from_stream_outcome.success()); + blob_content.append(iss.str()); + } + + auto get_blob_property_outcome = client.get_blob_property(container_name, blob_name); + REQUIRE(get_blob_property_outcome.success()); + REQUIRE(get_blob_property_outcome.response().size == 4 * 1024 * 1024 * 10); + + std::stringbuf strbuf; + std::ostream os(&strbuf); + auto get_blob_outcome = client.download_blob_to_stream(container_name, blob_name, 0, 4 * 1024 * 1024 * 10, os).get(); + REQUIRE(get_blob_outcome.success()); + REQUIRE(strbuf.str() == blob_content); + } + + SECTION("Append 5MB block unsuccessfully") + { + auto iss = as_test::get_istringstream_with_random_buffer(5 * 1024 * 1024); + auto append_block_from_stream_outcome = client.append_block_from_stream(container_name, blob_name, iss).get(); + REQUIRE(!append_block_from_stream_outcome.success()); + REQUIRE(append_block_from_stream_outcome.error().code == "413"); + REQUIRE(append_block_from_stream_outcome.error().code_name == "RequestBodyTooLarge"); + } + + SECTION("Append to non-existing blob unsuccessfully") + { + auto iss = as_test::get_istringstream_with_random_buffer(4 * 1024 * 1024); + auto append_block_from_stream_outcome = client.append_block_from_stream(container_name, as_test::get_random_string(20), iss).get(); + REQUIRE(!append_block_from_stream_outcome.success()); + REQUIRE(append_block_from_stream_outcome.error().code == "404"); + REQUIRE(append_block_from_stream_outcome.error().code_name == "BlobNotFound"); + } + + client.delete_container(container_name); +} diff --git a/test/integration/blob_container_integration_test.cpp b/test/integration/blob_container_integration_test.cpp new file mode 100644 index 0000000..c85a57b --- /dev/null +++ b/test/integration/blob_container_integration_test.cpp @@ -0,0 +1,218 @@ +#include "blob_integration_base.h" + +#include "../catch2/catch.hpp" + +TEST_CASE("Create containers", "[container],[blob_service]") +{ + azure::storage_lite::blob_client client = as_test::base::test_blob_client(); + std::string prefix = as_test::get_random_string(10); + SECTION("Create container with number and character name successfully") + { + auto container_name = prefix + as_test::get_random_string(10); + auto first_outcome = client.create_container(container_name).get(); + REQUIRE(first_outcome.success()); + auto second_outcome = client.get_container_property(container_name); + REQUIRE(second_outcome.success()); + REQUIRE_FALSE(second_outcome.response().etag.empty()); + client.delete_container(container_name).wait(); + } + + SECTION("Create container with uppercase name unsuccessfully") + { + auto container_name = "ABD" + prefix + as_test::get_random_string(10); + auto first_outcome = client.create_container(container_name).get(); + REQUIRE_FALSE(first_outcome.success()); + REQUIRE(first_outcome.error().code == "400"); + REQUIRE(first_outcome.error().code_name == "InvalidResourceName"); + auto second_outcome = client.get_container_property(container_name); + REQUIRE(second_outcome.success()); + REQUIRE(second_outcome.response().etag.empty()); + } + + SECTION("Create container with dash in name successfully") + { + auto container_name = prefix + "-" + as_test::get_random_string(10); + auto first_outcome = client.create_container(container_name).get(); + REQUIRE(first_outcome.success()); + auto second_outcome = client.get_container_property(container_name); + REQUIRE(second_outcome.success()); + REQUIRE_FALSE(second_outcome.response().etag.empty()); + client.delete_container(container_name).wait(); + } +} + +TEST_CASE("Delete containers", "[container],[blob_service]") +{ + azure::storage_lite::blob_client client = as_test::base::test_blob_client(); + std::string prefix = as_test::get_random_string(10); + SECTION("Delete existing container successfully") + { + auto container_name = prefix + as_test::get_random_string(10); + auto first_outcome = client.create_container(container_name).get(); + REQUIRE(first_outcome.success()); + auto second_outcome = client.get_container_property(container_name); + REQUIRE(second_outcome.success()); + REQUIRE_FALSE(second_outcome.response().etag.empty()); + auto third_outcome = client.delete_container(container_name).get(); + REQUIRE(third_outcome.success()); + } + + SECTION("Delete in-existing container successfully") + { + auto container_name = prefix + as_test::get_random_string(10); + auto first_outcome = client.delete_container(container_name).get(); + REQUIRE_FALSE(first_outcome.success()); + } +} + +TEST_CASE("Get Container Property", "[container], [blob_service]") +{ + azure::storage_lite::blob_client client = as_test::base::test_blob_client(); + std::string prefix = as_test::get_random_string(10); + SECTION("Get container property from existing container") + { + auto container_name = prefix + as_test::get_random_string(10); + auto first_outcome = client.create_container(container_name).get(); + REQUIRE(first_outcome.success()); + auto second_outcome = client.get_container_property(container_name); + REQUIRE(second_outcome.success()); + REQUIRE_FALSE(second_outcome.response().etag.empty()); + client.delete_container(container_name).wait(); + } + + SECTION("Get container property from non-existing container") + { + auto container_name = prefix + as_test::get_random_string(10); + auto first_outcome = client.get_container_property(container_name); + REQUIRE(first_outcome.success()); + } +} + +TEST_CASE("List containers segmented", "[container],[blob_service]") +{ + azure::storage_lite::blob_client client = as_test::base::test_blob_client(); + std::string prefix_1 = as_test::get_random_string(10); + std::string prefix_2 = as_test::get_random_string(10); + unsigned container_size = 5; + std::vector containers; + for (unsigned i = 0; i < container_size; ++i) + { + auto container_name_1 = prefix_1 + as_test::get_random_string(10); + auto container_name_2 = prefix_2 + as_test::get_random_string(10); + client.create_container(container_name_1).wait(); + client.create_container(container_name_2).wait(); + containers.push_back(container_name_1); + containers.push_back(container_name_2); + } + + SECTION("List containers successfully") { + auto list_containers_outcome = client.list_containers_segmented("", "", 10).get(); + REQUIRE(list_containers_outcome.success()); + auto result_containers = list_containers_outcome.response().containers; + REQUIRE(result_containers.size() == 10); + } + + SECTION("List containers with prefix successfully") + { + { + auto list_containers_outcome = client.list_containers_segmented(prefix_1, "", 5).get(); + REQUIRE(list_containers_outcome.success()); + REQUIRE(list_containers_outcome.response().next_marker.empty()); + auto result_containers = list_containers_outcome.response().containers; + REQUIRE(result_containers.size() == 5); + for (auto container : result_containers) + { + REQUIRE(std::find(containers.begin(), containers.end(), container.name) != containers.end()); + REQUIRE(container.name.substr(0, prefix_1.size()) == prefix_1); + } + } + + { + auto list_containers_outcome = client.list_containers_segmented(prefix_2, "", 5).get(); + REQUIRE(list_containers_outcome.success()); + REQUIRE(list_containers_outcome.response().next_marker.empty()); + auto result_containers = list_containers_outcome.response().containers; + REQUIRE(result_containers.size() == 5); + for (auto container : result_containers) + { + REQUIRE(std::find(containers.begin(), containers.end(), container.name) != containers.end()); + REQUIRE(container.name.substr(0, prefix_2.size()) == prefix_2); + } + } + + { + auto list_containers_outcome = client.list_containers_segmented(as_test::get_random_string(20), "").get(); + REQUIRE(list_containers_outcome.success()); + REQUIRE(list_containers_outcome.response().containers.size() == 0); + } + } + + SECTION("List containers with next marker successfully") + { + { + auto list_containers_outcome = client.list_containers_segmented(prefix_1, "", 3).get(); + REQUIRE(list_containers_outcome.success()); + REQUIRE(!list_containers_outcome.response().next_marker.empty()); + auto result_containers = list_containers_outcome.response().containers; + REQUIRE(result_containers.size() == 3); + for (auto container : result_containers) + { + REQUIRE(std::find(containers.begin(), containers.end(), container.name) != containers.end()); + REQUIRE(container.name.substr(0, prefix_1.size()) == prefix_1); + } + + auto second_list_containers_outcome = client.list_containers_segmented(prefix_1, list_containers_outcome.response().next_marker, 2).get(); + REQUIRE(second_list_containers_outcome.response().next_marker.empty()); + auto second_result_containers = second_list_containers_outcome.response().containers; + REQUIRE(second_result_containers.size() == 2); + for (auto container : second_result_containers) + { + REQUIRE(std::find(containers.begin(), containers.end(), container.name) != containers.end()); + REQUIRE(container.name.substr(0, prefix_1.size()) == prefix_1); + } + } + + { + auto list_containers_outcome = client.list_containers_segmented(prefix_2, "", 3).get(); + REQUIRE(list_containers_outcome.success()); + REQUIRE(!list_containers_outcome.response().next_marker.empty()); + auto result_containers = list_containers_outcome.response().containers; + REQUIRE(result_containers.size() == 3); + for (auto container : result_containers) + { + REQUIRE(std::find(containers.begin(), containers.end(), container.name) != containers.end()); + REQUIRE(container.name.substr(0, prefix_1.size()) == prefix_2); + } + + auto second_list_containers_outcome = client.list_containers_segmented(prefix_2, list_containers_outcome.response().next_marker, 2).get(); + REQUIRE(second_list_containers_outcome.response().next_marker.empty()); + auto second_result_containers = second_list_containers_outcome.response().containers; + REQUIRE(second_result_containers.size() == 2); + for (auto container : second_result_containers) + { + REQUIRE(std::find(containers.begin(), containers.end(), container.name) != containers.end()); + REQUIRE(container.name.substr(0, prefix_1.size()) == prefix_2); + } + } + } + + SECTION("List containers with invalid prefix successfully") + { + auto list_containers_outcome = client.list_containers_segmented("1~invalid~~%d_prefix", "").get(); + REQUIRE(list_containers_outcome.success()); + REQUIRE(list_containers_outcome.response().containers.empty()); + } + + SECTION("List containers with invalid next marker unsuccessfully") + { + auto list_containers_outcome = client.list_containers_segmented("", "1~invalid~~%d_continuation_token").get(); + REQUIRE(!list_containers_outcome.success()); + REQUIRE(list_containers_outcome.error().code == "400"); + REQUIRE(list_containers_outcome.error().code_name == "OutOfRangeInput"); + } + + for (auto container : containers) + { + client.delete_container(container).wait(); + } +} diff --git a/test/integration/blob_general_integration_test.cpp b/test/integration/blob_general_integration_test.cpp new file mode 100644 index 0000000..6882169 --- /dev/null +++ b/test/integration/blob_general_integration_test.cpp @@ -0,0 +1,250 @@ +#include "blob_integration_base.h" + +#include "../catch2/catch.hpp" + +// List all blobs that returns a iterator is going to be supported in the future, and this test case set will be valid again. + +//TEST_CASE("List blobs", "[blob],[blob_service]") +//{ +// azure::storage_lite::blob_client client = as_test::base::test_blob_client(); +// std::string container_name = as_test::create_random_container("", client); +// unsigned blob_count = 15; +// std::vector blobs; +// std::string blob_prefix_1 = as_test::get_random_string(20); +// std::string blob_prefix_2 = as_test::get_random_string(20); +// for (unsigned i = 0; i < blob_count; ++i) +// { +// auto blob_name_1 = blob_prefix_1 + as_test::get_random_string(10); +// auto blob_name_2 = blob_prefix_2 + as_test::get_random_string(10); +// { +// auto create_page_blob_outcome = client.create_page_blob(container_name, blob_name_1, 512).get(); +// REQUIRE(create_page_blob_outcome.success()); +// blobs.push_back(blob_name_1); +// } +// { +// auto create_page_blob_outcome = client.create_page_blob(container_name, blob_name_2, 1024).get(); +// REQUIRE(create_page_blob_outcome.success()); +// blobs.push_back(blob_name_2); +// } +// } +// +// SECTION("List Blobs successfully") { +// auto list_blob_outcome = client.list_blobs(container_name, "").get(); +// REQUIRE(list_blob_outcome.success()); +// REQUIRE(!list_blob_outcome.response().next_marker.empty()); +// auto listed_blobs = list_blob_outcome.response().blobs; +// REQUIRE(listed_blobs.size() == 2);// list_blobs does not do what it said to do. +// for (auto blob : listed_blobs) +// { +// REQUIRE(((blob.content_length == 1024) || (blob.content_length == 512))); +// REQUIRE(std::find(blobs.begin(), blobs.end(), blob.name) != blobs.end()); +// } +// } +// +// SECTION("List Blobs with prefix successfully") { +// { +// auto list_blob_outcome = client.list_blobs(container_name, blob_prefix_1).get(); +// REQUIRE(list_blob_outcome.success()); +// REQUIRE(!list_blob_outcome.response().next_marker.empty()); +// auto listed_blobs = list_blob_outcome.response().blobs; +// REQUIRE(listed_blobs.size() == 2); +// for (auto blob : listed_blobs) +// { +// REQUIRE(blob.content_length == 512); +// REQUIRE(std::find(blobs.begin(), blobs.end(), blob.name) != blobs.end()); +// } +// } +// +// { +// auto list_blob_outcome = client.list_blobs(container_name, blob_prefix_2).get(); +// REQUIRE(list_blob_outcome.success()); +// REQUIRE(!list_blob_outcome.response().next_marker.empty()); +// auto listed_blobs = list_blob_outcome.response().blobs; +// REQUIRE(listed_blobs.size() == 2); +// for (auto blob : listed_blobs) +// { +// REQUIRE(blob.content_length == 1024); +// REQUIRE(std::find(blobs.begin(), blobs.end(), blob.name) != blobs.end()); +// } +// } +// } +// +// SECTION("List Blobs with invalid prefix successfully") { +// auto list_blob_outcome = client.list_blobs(container_name, "1~invalid~~%d_prefix").get(); +// REQUIRE(list_blob_outcome.success()); +// REQUIRE(list_blob_outcome.response().next_marker.empty()); +// REQUIRE(list_blob_outcome.response().blobs.empty()); +// } +// +// client.delete_container(container_name); +//} + +TEST_CASE("List blobs segmented", "[blob],[blob_service]") +{ + azure::storage_lite::blob_client client = as_test::base::test_blob_client(); + std::string container_name = as_test::create_random_container("", client); + unsigned blob_count = 15; + std::vector blobs; + std::string blob_prefix_1 = as_test::get_random_string(20); + std::string blob_prefix_2 = as_test::get_random_string(20); + for (unsigned i = 0; i < blob_count; ++i) + { + auto blob_name_1 = blob_prefix_1 + "/" + as_test::get_random_string(10); + auto blob_name_2 = blob_prefix_2 + "-" + as_test::get_random_string(10); + { + auto create_page_blob_outcome = client.create_page_blob(container_name, blob_name_1, 512).get(); + REQUIRE(create_page_blob_outcome.success()); + blobs.push_back(blob_name_1); + } + { + auto create_page_blob_outcome = client.create_page_blob(container_name, blob_name_2, 1024).get(); + REQUIRE(create_page_blob_outcome.success()); + blobs.push_back(blob_name_2); + } + } + + SECTION("List blobs segmented successfully") + { + auto list_blob_outcome = client.list_blobs_segmented(container_name, "", "", "").get(); + REQUIRE(list_blob_outcome.success()); + auto marker = list_blob_outcome.response().next_marker; + auto listed_blobs = list_blob_outcome.response().blobs; + while (!marker.empty()) + { + auto list_again_blob_outcome = client.list_blobs_segmented(container_name, "", marker, "").get(); + REQUIRE(list_again_blob_outcome.success()); + auto more_blobs = list_again_blob_outcome.response().blobs; + listed_blobs.reserve(listed_blobs.size() + more_blobs.size()); + listed_blobs.insert(listed_blobs.end(), more_blobs.begin(), more_blobs.end()); + marker = list_again_blob_outcome.response().next_marker; + } + REQUIRE(marker.empty()); + REQUIRE(listed_blobs.size() == 30); + for (auto blob : listed_blobs) + { + REQUIRE(((blob.content_length == 1024) || (blob.content_length == 512))); + REQUIRE(std::find(blobs.begin(), blobs.end(), blob.name) != blobs.end()); + } + } + + SECTION("List blobs segmented with prefix successfully") + { + { + auto list_blob_outcome = client.list_blobs_segmented(container_name, "", "", blob_prefix_1, 2).get(); + REQUIRE(list_blob_outcome.success()); + auto marker = list_blob_outcome.response().next_marker; + auto listed_blobs = list_blob_outcome.response().blobs; + while (!marker.empty()) + { + auto list_again_blob_outcome = client.list_blobs_segmented(container_name, "", marker, blob_prefix_1, 2).get(); + REQUIRE(list_again_blob_outcome.success()); + auto more_blobs = list_again_blob_outcome.response().blobs; + listed_blobs.reserve(listed_blobs.size() + more_blobs.size()); + listed_blobs.insert(listed_blobs.end(), more_blobs.begin(), more_blobs.end()); + marker = list_again_blob_outcome.response().next_marker; + } + REQUIRE(marker.empty()); + REQUIRE(listed_blobs.size() == 15); + for (auto blob : listed_blobs) + { + REQUIRE(blob.content_length == 512); + REQUIRE(std::find(blobs.begin(), blobs.end(), blob.name) != blobs.end()); + } + } + + { + auto list_blob_outcome = client.list_blobs_segmented(container_name, "", "", blob_prefix_2, 2).get(); + REQUIRE(list_blob_outcome.success()); + auto marker = list_blob_outcome.response().next_marker; + auto listed_blobs = list_blob_outcome.response().blobs; + while (!marker.empty()) + { + auto list_again_blob_outcome = client.list_blobs_segmented(container_name, "", marker, blob_prefix_2, 2).get(); + REQUIRE(list_again_blob_outcome.success()); + auto more_blobs = list_again_blob_outcome.response().blobs; + listed_blobs.reserve(listed_blobs.size() + more_blobs.size()); + listed_blobs.insert(listed_blobs.end(), more_blobs.begin(), more_blobs.end()); + marker = list_again_blob_outcome.response().next_marker; + } + REQUIRE(marker.empty()); + REQUIRE(listed_blobs.size() == 15); + for (auto blob : listed_blobs) + { + REQUIRE(blob.content_length == 1024); + REQUIRE(std::find(blobs.begin(), blobs.end(), blob.name) != blobs.end()); + } + } + } + client.delete_container(container_name); +} + +TEST_CASE("Get blob property", "[blob],[blob_service]") +{ + azure::storage_lite::blob_client client = as_test::base::test_blob_client(); + std::string container_name = as_test::create_random_container("", client); + std::string blob_name = as_test::get_random_string(20); + auto create_blob_outcome = client.create_page_blob(container_name, blob_name, 1024).get(); + REQUIRE(create_blob_outcome.success()); + + SECTION("Get blob property for existing blob successfully") + { + auto get_blob_property_outcome = client.get_blob_property(container_name, blob_name); + REQUIRE(get_blob_property_outcome.success()); + REQUIRE(get_blob_property_outcome.response().size == 1024); + } + + SECTION("Get blob property for non-existing blob successfully") + { + auto get_blob_property_outcome = client.get_blob_property(container_name, as_test::get_random_string(20)); + REQUIRE(get_blob_property_outcome.success()); + } + + client.delete_container(container_name); +} + +TEST_CASE("Delete blob", "[blob],[blob_service]") +{ + azure::storage_lite::blob_client client = as_test::base::test_blob_client(); + std::string container_name = as_test::create_random_container("", client); + std::string blob_name = as_test::get_random_string(20); + auto create_blob_outcome = client.create_page_blob(container_name, blob_name, 1024).get(); + REQUIRE(create_blob_outcome.success()); + + SECTION("Delete existing blob successfully") + { + auto delete_blob_outcome = client.delete_blob(container_name, blob_name).get(); + REQUIRE(delete_blob_outcome.success()); + } + + SECTION("Delete non-existing blob successfully") + { + auto delete_blob_outcome = client.delete_blob(container_name, as_test::get_random_string(20)).get(); + REQUIRE(!delete_blob_outcome.success()); + REQUIRE(delete_blob_outcome.error().code == "404"); + REQUIRE(delete_blob_outcome.error().code_name == "BlobNotFound"); + } + + client.delete_container(container_name); +} + +TEST_CASE("Start copy blob", "[blob],[blob_service]") +{ + azure::storage_lite::blob_client client = as_test::base::test_blob_client(); + std::string container_name = as_test::create_random_container("", client); + std::string blob_name = as_test::get_random_string(20); + auto iss = as_test::get_istringstream_with_random_buffer(2048); + auto create_blob_outcome = client.upload_block_blob_from_stream(container_name, blob_name, iss, std::vector>()).get(); + REQUIRE(create_blob_outcome.success()); + + SECTION("Start copy blob successfully") + { + std::string dest_blob_name = as_test::get_random_string(20); + auto start_copy_outcome = client.start_copy(container_name, blob_name, container_name, dest_blob_name).get(); + REQUIRE(start_copy_outcome.success()); + auto get_blob_property_outcome = client.get_blob_property(container_name, dest_blob_name); + REQUIRE(get_blob_property_outcome.success()); + REQUIRE(get_blob_property_outcome.response().size == 2048); + } + + client.delete_container(container_name); +} diff --git a/test/integration/blob_integration_base.cpp b/test/integration/blob_integration_base.cpp new file mode 100644 index 0000000..ae5fa80 --- /dev/null +++ b/test/integration/blob_integration_base.cpp @@ -0,0 +1,35 @@ +#include "blob_integration_base.h" + +#include "../test_constants.h" + +namespace as_test { + std::string create_random_container(const std::string& prefix, azure::storage_lite::blob_client& client) + { + //Assume prefix is less than max prefix size. + auto container_name = prefix + get_random_string(MAX_PREFIX_SIZE - prefix.size()); + auto result = client.create_container(container_name).get(); + if (!result.success()) + { + return create_random_container(prefix, client); + } + return container_name; + } + + std::vector create_random_containers(const std::string& prefix, azure::storage_lite::blob_client& client, size_t count) + { + std::vector results; + for (size_t i = 0; i < count; ++i) + { + results.push_back(create_random_container(prefix, client)); + } + return results; + } + + std::string get_base64_block_id(unsigned id) + { + std::string raw_block_id = std::to_string(id); + //pad the string to length of 6. + raw_block_id.insert(raw_block_id.begin(), 6 - raw_block_id.length(), '0'); + return to_base64(raw_block_id.c_str(), 6); + } +} diff --git a/test/integration/blob_integration_base.h b/test/integration/blob_integration_base.h new file mode 100644 index 0000000..4f33066 --- /dev/null +++ b/test/integration/blob_integration_base.h @@ -0,0 +1,11 @@ +#pragma once + +#include "../test_base.h" + +namespace as_test { + std::string create_random_container(const std::string& prefix, azure::storage_lite::blob_client& client); + + std::vector create_random_containers(const std::string& prefix, azure::storage_lite::blob_client& client, size_t count); + + std::string get_base64_block_id(unsigned id); +} diff --git a/test/integration/block_blob_integration_test.cpp b/test/integration/block_blob_integration_test.cpp new file mode 100644 index 0000000..304fe2d --- /dev/null +++ b/test/integration/block_blob_integration_test.cpp @@ -0,0 +1,347 @@ +#include "blob_integration_base.h" + +#include "../catch2/catch.hpp" + +TEST_CASE("Upload block blob from stream", "[block blob],[blob_service]") +{ + azure::storage_lite::blob_client client = as_test::base::test_blob_client(); + std::string container_name = as_test::create_random_container("", client); + std::string blob_name = as_test::get_random_string(20); + + SECTION("Upload block blob from a 2048 byte stream successfully") + { + auto iss = as_test::get_istringstream_with_random_buffer(2048); + auto create_blob_outcome = client.upload_block_blob_from_stream(container_name, blob_name, iss, std::vector>()).get(); + REQUIRE(create_blob_outcome.success()); + std::stringbuf strbuf; + std::ostream os(&strbuf); + auto get_blob_outcome = client.download_blob_to_stream(container_name, blob_name, 0, 4096, os).get(); + REQUIRE(get_blob_outcome.success()); + REQUIRE(strbuf.str() == iss.str()); + } + + SECTION("Upload block blob from a 64MB stream successfully") + { + auto iss = as_test::get_istringstream_with_random_buffer(64 * 1024 * 1024); + auto create_blob_outcome = client.upload_block_blob_from_stream(container_name, blob_name, iss, std::vector>()).get(); + REQUIRE(create_blob_outcome.success()); + std::stringbuf strbuf; + std::ostream os(&strbuf); + auto get_blob_outcome = client.download_blob_to_stream(container_name, blob_name, 0, 64 * 1024 * 1024, os).get(); + REQUIRE(get_blob_outcome.success()); + REQUIRE(strbuf.str() == iss.str()); + } + + SECTION("Upload block blob from a 257MB stream unsuccessfully") + { + auto iss = as_test::get_istringstream_with_random_buffer(257 * 1024 * 1024); + auto create_blob_outcome = client.upload_block_blob_from_stream(container_name, blob_name, iss, std::vector>()).get(); + REQUIRE(!create_blob_outcome.success()); + } + + SECTION("Upload block blob from with metadata successfully") + { + auto iss = as_test::get_istringstream_with_random_buffer(2048); + std::vector> meta; + meta.push_back(std::make_pair(std::string("custommeta1"), std::string("meta1"))); + meta.push_back(std::make_pair(std::string("custommeta2"), std::string("meta2"))); + auto create_blob_outcome = client.upload_block_blob_from_stream(container_name, blob_name, iss, meta).get(); + REQUIRE(create_blob_outcome.success()); + auto get_blob_property_outcome = client.get_blob_property(container_name, blob_name); + REQUIRE(get_blob_property_outcome.success()); + for ( auto m : get_blob_property_outcome.response().metadata ) + { + if (m.first == "custommeta1") + { + REQUIRE(m.second == "meta1"); + } + else + { + REQUIRE(m.first == "custommeta2"); + REQUIRE(m.second == "meta2"); + + } + } + } + + client.delete_container(container_name); +} + +TEST_CASE("Upload blob block from stream", "[block blob],[blob_service]") +{ + azure::storage_lite::blob_client client = as_test::base::test_blob_client(); + std::string container_name = as_test::create_random_container("", client); + std::string blob_name = as_test::get_random_string(20); + + SECTION("Upload block from stream successfully") + { + for (unsigned i = 0; i < 10; ++i) + { + auto iss = as_test::get_istringstream_with_random_buffer(4 * 1024 * 1024); + std::string block_id = as_test::get_base64_block_id(i); + auto upload_block_outcome = client.upload_block_from_stream(container_name, blob_name, block_id, iss).get(); + REQUIRE(upload_block_outcome.success()); + } + } + + SECTION("Upload block from stream with invalid block ID unsuccessfully") + { + auto iss = as_test::get_istringstream_with_random_buffer(4 * 1024 * 1024); + std::string block_id = "000001"; + auto upload_block_outcome = client.upload_block_from_stream(container_name, blob_name, block_id, iss).get(); + REQUIRE(!upload_block_outcome.success()); + } + + client.delete_container(container_name); +} + +TEST_CASE("Put block list", "[block blob],[blob_service]") +{ + azure::storage_lite::blob_client client = as_test::base::test_blob_client(); + std::string container_name = as_test::create_random_container("", client); + std::string blob_name = as_test::get_random_string(20); + + SECTION("Put block list with all blocks uncommitted successfully") + { + std::vector block_list; + for (unsigned i = 0; i < 10; ++i) + { + azure::storage_lite::put_block_list_request_base::block_item item; + auto iss = as_test::get_istringstream_with_random_buffer(4 * 1024 * 1024); + item.id = as_test::get_base64_block_id(i); + auto upload_block_outcome = client.upload_block_from_stream(container_name, blob_name, item.id, iss).get(); + REQUIRE(upload_block_outcome.success()); + item.type = azure::storage_lite::put_block_list_request_base::block_type::uncommitted; + block_list.push_back(item); + } + + auto put_block_list_outcome = client.put_block_list(container_name, blob_name, block_list, std::vector>()).get(); + REQUIRE(put_block_list_outcome.success()); + auto get_block_list_outcome = client.get_block_list(container_name, blob_name).get(); + REQUIRE(get_block_list_outcome.success()); + auto committed_block_list = get_block_list_outcome.response().committed; + auto uncommitted_block_list = get_block_list_outcome.response().uncommitted; + REQUIRE(committed_block_list.size() == 10); + REQUIRE(uncommitted_block_list.size() == 0); + auto get_blob_property_outcome = client.get_blob_property(container_name, blob_name); + REQUIRE(get_blob_property_outcome.success()); + REQUIRE(get_blob_property_outcome.response().size == 4 * 1024 * 1024 * 10); + } + + SECTION("Put block list with all blocks committed successfully") + { + std::vector block_list; + for (unsigned i = 0; i < 10; ++i) + { + azure::storage_lite::put_block_list_request_base::block_item item; + auto iss = as_test::get_istringstream_with_random_buffer(4 * 1024 * 1024); + item.id = as_test::get_base64_block_id(i); + auto upload_block_outcome = client.upload_block_from_stream(container_name, blob_name, item.id, iss).get(); + REQUIRE(upload_block_outcome.success()); + item.type = azure::storage_lite::put_block_list_request_base::block_type::uncommitted; + block_list.push_back(item); + } + { + auto put_block_list_outcome = client.put_block_list(container_name, blob_name, block_list, std::vector>()).get(); + REQUIRE(put_block_list_outcome.success()); + auto get_block_list_outcome = client.get_block_list(container_name, blob_name).get(); + REQUIRE(get_block_list_outcome.success()); + auto committed_block_list = get_block_list_outcome.response().committed; + auto uncommitted_block_list = get_block_list_outcome.response().uncommitted; + REQUIRE(committed_block_list.size() == 10); + REQUIRE(uncommitted_block_list.size() == 0); + auto get_blob_property_outcome = client.get_blob_property(container_name, blob_name); + REQUIRE(get_blob_property_outcome.success()); + REQUIRE(get_blob_property_outcome.response().size == 4 * 1024 * 1024 * 10); + } + for (unsigned i = 0; i < 10; ++i) + { + block_list[i].type = azure::storage_lite::put_block_list_request_base::block_type::committed; + } + { + block_list.pop_back(); + auto put_block_list_outcome = client.put_block_list(container_name, blob_name, block_list, std::vector>()).get(); + REQUIRE(put_block_list_outcome.success()); + auto get_block_list_outcome = client.get_block_list(container_name, blob_name).get(); + REQUIRE(get_block_list_outcome.success()); + auto committed_block_list = get_block_list_outcome.response().committed; + auto uncommitted_block_list = get_block_list_outcome.response().uncommitted; + REQUIRE(committed_block_list.size() == 9); + REQUIRE(uncommitted_block_list.size() == 0); + + auto get_blob_property_outcome = client.get_blob_property(container_name, blob_name); + REQUIRE(get_blob_property_outcome.success()); + REQUIRE(get_blob_property_outcome.response().size == 4 * 1024 * 1024 * 9); + } + } + + SECTION("Put block list with both committed and uncommitted blocks successfully") + { + std::vector block_list; + for (unsigned i = 0; i < 5; ++i) + { + azure::storage_lite::put_block_list_request_base::block_item item; + auto iss = as_test::get_istringstream_with_random_buffer(4 * 1024 * 1024); + item.id = as_test::get_base64_block_id(i); + auto upload_block_outcome = client.upload_block_from_stream(container_name, blob_name, item.id, iss).get(); + REQUIRE(upload_block_outcome.success()); + item.type = azure::storage_lite::put_block_list_request_base::block_type::uncommitted; + block_list.push_back(item); + } + { + auto put_block_list_outcome = client.put_block_list(container_name, blob_name, block_list, std::vector>()).get(); + REQUIRE(put_block_list_outcome.success()); + auto get_block_list_outcome = client.get_block_list(container_name, blob_name).get(); + REQUIRE(get_block_list_outcome.success()); + auto committed_block_list = get_block_list_outcome.response().committed; + auto uncommitted_block_list = get_block_list_outcome.response().uncommitted; + REQUIRE(committed_block_list.size() == 5); + REQUIRE(uncommitted_block_list.size() == 0); + auto get_blob_property_outcome = client.get_blob_property(container_name, blob_name); + REQUIRE(get_blob_property_outcome.success()); + REQUIRE(get_blob_property_outcome.response().size == 4 * 1024 * 1024 * 5); + } + for (unsigned i = 0; i < 5; ++i) + { + block_list[i].type = azure::storage_lite::put_block_list_request_base::block_type::committed; + } + block_list.pop_back(); + for (unsigned i = 5; i < 10; ++i) + { + azure::storage_lite::put_block_list_request_base::block_item item; + auto iss = as_test::get_istringstream_with_random_buffer(4 * 1024 * 1024); + item.id = as_test::get_base64_block_id(i); + auto upload_block_outcome = client.upload_block_from_stream(container_name, blob_name, item.id, iss).get(); + REQUIRE(upload_block_outcome.success()); + item.type = azure::storage_lite::put_block_list_request_base::block_type::uncommitted; + block_list.push_back(item); + } + { + auto put_block_list_outcome = client.put_block_list(container_name, blob_name, block_list, std::vector>()).get(); + REQUIRE(put_block_list_outcome.success()); + auto get_block_list_outcome = client.get_block_list(container_name, blob_name).get(); + REQUIRE(get_block_list_outcome.success()); + auto committed_block_list = get_block_list_outcome.response().committed; + auto uncommitted_block_list = get_block_list_outcome.response().uncommitted; + REQUIRE(committed_block_list.size() == 9); + REQUIRE(uncommitted_block_list.size() == 0); + + auto get_blob_property_outcome = client.get_blob_property(container_name, blob_name); + REQUIRE(get_blob_property_outcome.success()); + REQUIRE(get_blob_property_outcome.response().size == 4 * 1024 * 1024 * 9); + } + } + + SECTION("Put block list with invalid block list unsuccessfully") + { + std::vector block_list; + for (unsigned i = 0; i < 10; ++i) + { + azure::storage_lite::put_block_list_request_base::block_item item; + auto iss = as_test::get_istringstream_with_random_buffer(4 * 1024 * 1024); + item.id = as_test::get_base64_block_id(i); + auto upload_block_outcome = client.upload_block_from_stream(container_name, blob_name, item.id, iss).get(); + REQUIRE(upload_block_outcome.success()); + item.type = azure::storage_lite::put_block_list_request_base::block_type::committed; + block_list.push_back(item); + } + + auto put_block_list_outcome = client.put_block_list(container_name, blob_name, block_list, std::vector>()).get(); + REQUIRE(!put_block_list_outcome.success()); + } + + client.delete_container(container_name); +} + +TEST_CASE("Get block list", "[block blob],[blob_service]") +{ + azure::storage_lite::blob_client client = as_test::base::test_blob_client(); + std::string container_name = as_test::create_random_container("", client); + std::string blob_name = as_test::get_random_string(20); + + SECTION("Get committed block list successfully") + { + std::vector block_list; + for (unsigned i = 0; i < 10; ++i) + { + azure::storage_lite::put_block_list_request_base::block_item item; + auto iss = as_test::get_istringstream_with_random_buffer(4 * 1024 * 1024); + item.id = as_test::get_base64_block_id(i); + auto upload_block_outcome = client.upload_block_from_stream(container_name, blob_name, item.id, iss).get(); + REQUIRE(upload_block_outcome.success()); + item.type = azure::storage_lite::put_block_list_request_base::block_type::uncommitted; + block_list.push_back(item); + } + + auto put_block_list_outcome = client.put_block_list(container_name, blob_name, block_list, std::vector>()).get(); + REQUIRE(put_block_list_outcome.success()); + + auto get_block_list_outcome = client.get_block_list(container_name, blob_name).get(); + REQUIRE(get_block_list_outcome.success()); + auto committed_block_list = get_block_list_outcome.response().committed; + auto uncommitted_block_list = get_block_list_outcome.response().uncommitted; + REQUIRE(committed_block_list.size() == 10); + REQUIRE(uncommitted_block_list.size() == 0); + } + + SECTION("Get un-committed block list successfully") + { + for (unsigned i = 0; i < 10; ++i) + { + auto iss = as_test::get_istringstream_with_random_buffer(4 * 1024 * 1024); + std::string block_id = as_test::get_base64_block_id(i); + auto upload_block_outcome = client.upload_block_from_stream(container_name, blob_name, block_id, iss).get(); + REQUIRE(upload_block_outcome.success()); + } + + auto get_block_list_outcome = client.get_block_list(container_name, blob_name).get(); + REQUIRE(get_block_list_outcome.success()); + auto committed_block_list = get_block_list_outcome.response().committed; + auto uncommitted_block_list = get_block_list_outcome.response().uncommitted; + REQUIRE(committed_block_list.size() == 0); + REQUIRE(uncommitted_block_list.size() == 10); + } + + SECTION("Get both committed and uncommitted block list successfully") + { + std::vector block_list; + for (unsigned i = 0; i < 10; ++i) + { + azure::storage_lite::put_block_list_request_base::block_item item; + auto iss = as_test::get_istringstream_with_random_buffer(4 * 1024 * 1024); + item.id = as_test::get_base64_block_id(i); + auto upload_block_outcome = client.upload_block_from_stream(container_name, blob_name, item.id, iss).get(); + REQUIRE(upload_block_outcome.success()); + item.type = azure::storage_lite::put_block_list_request_base::block_type::uncommitted; + block_list.push_back(item); + } + + auto put_block_list_outcome = client.put_block_list(container_name, blob_name, block_list, std::vector>()).get(); + REQUIRE(put_block_list_outcome.success()); + + for (unsigned i = 0; i < 10; ++i) + { + auto iss = as_test::get_istringstream_with_random_buffer(4 * 1024 * 1024); + auto block_id = as_test::get_base64_block_id(i); + auto upload_block_outcome = client.upload_block_from_stream(container_name, blob_name, block_id, iss).get(); + REQUIRE(upload_block_outcome.success()); + } + + auto get_block_list_outcome = client.get_block_list(container_name, blob_name).get(); + REQUIRE(get_block_list_outcome.success()); + auto committed_block_list = get_block_list_outcome.response().committed; + auto uncommitted_block_list = get_block_list_outcome.response().uncommitted; + REQUIRE(committed_block_list.size() == 10); + REQUIRE(uncommitted_block_list.size() == 10); + } + + SECTION("Get empty block list successfully") + { + auto get_block_list_outcome = client.get_block_list(container_name, blob_name).get(); + REQUIRE(!get_block_list_outcome.success()); + REQUIRE(get_block_list_outcome.error().code == "404"); + REQUIRE(get_block_list_outcome.error().code_name == "BlobNotFound"); + } + + client.delete_container(container_name); +} \ No newline at end of file diff --git a/test/integration/page_blob_integration_test.cpp b/test/integration/page_blob_integration_test.cpp new file mode 100644 index 0000000..46f5eb3 --- /dev/null +++ b/test/integration/page_blob_integration_test.cpp @@ -0,0 +1,254 @@ +#include "blob_integration_base.h" + +#include "../catch2/catch.hpp" + +TEST_CASE("Create page blob", "[page blob],[blob_service]") +{ + azure::storage_lite::blob_client client = as_test::base::test_blob_client(); + std::string container_name = as_test::create_random_container("", client); + std::string blob_name = as_test::get_random_string(20); + + SECTION("Create page blob with valid name successfully") + { + auto create_page_blob_outcome = client.create_page_blob(container_name, blob_name, 1024).get(); + REQUIRE(create_page_blob_outcome.success()); + } + + SECTION("Create page blob with invalid container unsuccessfully") + { + auto create_page_blob_outcome = client.create_page_blob(as_test::get_random_string(20), blob_name, 1024).get(); + REQUIRE(!create_page_blob_outcome.success()); + } + + SECTION("Create page blob with too large a size unsuccessfully") + { + long long size = 1024ll * 1024ll * 1024ll * 8ll + 1ll; + auto create_page_blob_outcome = client.create_page_blob(container_name, blob_name, size).get(); + REQUIRE(!create_page_blob_outcome.success()); + } + + SECTION("Create 8TB page blob successfully") + { + long long size = 1024ll * 1024ll * 1024ll * 8ll; + auto create_page_blob_outcome = client.create_page_blob(container_name, blob_name, size).get(); + REQUIRE(create_page_blob_outcome.success()); + } + + client.delete_container(container_name); +} + +TEST_CASE("Put page from stream", "[page blob],[blob_service]") +{ + azure::storage_lite::blob_client client = as_test::base::test_blob_client(); + std::string container_name = as_test::create_random_container("", client); + std::string blob_name = as_test::get_random_string(20); + auto create_page_blob_outcome = client.create_page_blob(container_name, blob_name, 64 * 1024 * 1024).get(); + REQUIRE(create_page_blob_outcome.success()); + + SECTION("Put 4MB page successfully") + { + auto iss = as_test::get_istringstream_with_random_buffer(4 * 1024 * 1024); + auto put_page_from_stream_outcome = client.put_page_from_stream(container_name, blob_name, 0, iss.str().size(), iss).get(); + REQUIRE(put_page_from_stream_outcome.success()); + + auto get_blob_property_outcome = client.get_blob_property(container_name, blob_name); + REQUIRE(get_blob_property_outcome.success()); + REQUIRE(get_blob_property_outcome.response().size == 64 * 1024 * 1024); + + std::stringbuf strbuf; + std::ostream os(&strbuf); + auto get_blob_outcome = client.download_blob_to_stream(container_name, blob_name, 0, 4 * 1024 * 1024, os).get(); + REQUIRE(get_blob_outcome.success()); + REQUIRE(strbuf.str() == iss.str()); + } + + SECTION("Put 5MB page unsuccessfully") + { + auto iss = as_test::get_istringstream_with_random_buffer(5 * 1024 * 1024); + auto put_page_from_stream_outcome = client.put_page_from_stream(container_name, blob_name, 0, iss.str().size(), iss).get(); + REQUIRE(!put_page_from_stream_outcome.success()); + } + + SECTION("Put 4MB page to an 512 byte aligned offset successfully") + { + auto iss = as_test::get_istringstream_with_random_buffer(4 * 1024 * 1024); + auto put_page_from_stream_outcome = client.put_page_from_stream(container_name, blob_name, 1024, iss.str().size(), iss).get(); + REQUIRE(put_page_from_stream_outcome.success()); + + auto get_blob_property_outcome = client.get_blob_property(container_name, blob_name); + REQUIRE(get_blob_property_outcome.success()); + REQUIRE(get_blob_property_outcome.response().size == 64 * 1024 * 1024); + + std::stringbuf strbuf; + std::ostream os(&strbuf); + auto get_blob_outcome = client.download_blob_to_stream(container_name, blob_name, 1024, 4 * 1024 * 1024, os).get(); + REQUIRE(get_blob_outcome.success()); + REQUIRE(strbuf.str() == iss.str()); + } + + SECTION("Put 4MB page to an 512 not aligned offset unsuccessfully") + { + auto iss = as_test::get_istringstream_with_random_buffer(4 * 1024 * 1024); + auto put_page_from_stream_outcome = client.put_page_from_stream(container_name, blob_name, 511, iss.str().size(), iss).get(); + REQUIRE(!put_page_from_stream_outcome.success()); + } + + SECTION("Put 4MB page to same pages overwrites the data successfully") + { + auto iss1 = as_test::get_istringstream_with_random_buffer(4 * 1024 * 1024); + auto put_page_from_stream_outcome = client.put_page_from_stream(container_name, blob_name, 1024, iss1.str().size(), iss1).get(); + REQUIRE(put_page_from_stream_outcome.success()); + + auto iss2 = as_test::get_istringstream_with_random_buffer(4 * 1024 * 1024); + put_page_from_stream_outcome = client.put_page_from_stream(container_name, blob_name, 1024, iss2.str().size(), iss2).get(); + REQUIRE(put_page_from_stream_outcome.success()); + + auto get_blob_property_outcome = client.get_blob_property(container_name, blob_name); + REQUIRE(get_blob_property_outcome.success()); + REQUIRE(get_blob_property_outcome.response().size == 64 * 1024 * 1024); + + std::stringbuf strbuf; + std::ostream os(&strbuf); + auto get_blob_outcome = client.download_blob_to_stream(container_name, blob_name, 1024, 4 * 1024 * 1024, os).get(); + REQUIRE(get_blob_outcome.success()); + REQUIRE(strbuf.str() == iss2.str()); + } + + SECTION("Put 4MB page exceeds the size of the page blob unsuccessfully") + { + auto iss = as_test::get_istringstream_with_random_buffer(4 * 1024 * 1024); + auto put_page_from_stream_outcome = client.put_page_from_stream(container_name, blob_name, 62 * 1024 * 1024, iss.str().size(), iss).get(); + REQUIRE(!put_page_from_stream_outcome.success()); + } + + client.delete_container(container_name); +} + +TEST_CASE("Clear page", "[page blob],[blob_service]") +{ + azure::storage_lite::blob_client client = as_test::base::test_blob_client(); + std::string container_name = as_test::create_random_container("", client); + std::string blob_name = as_test::get_random_string(20); + auto create_page_blob_outcome = client.create_page_blob(container_name, blob_name, 4 * 1024 * 1024).get(); + REQUIRE(create_page_blob_outcome.success()); + auto iss = as_test::get_istringstream_with_random_buffer(4 * 1024 * 1024); + auto put_page_from_stream_outcome = client.put_page_from_stream(container_name, blob_name, 0, iss.str().size(), iss).get(); + REQUIRE(put_page_from_stream_outcome.success()); + + SECTION("Clear 4MB page successfully") + { + auto clear_page_outcome = client.clear_page(container_name, blob_name, 0, 4 * 1024 * 1024).get(); + REQUIRE(clear_page_outcome.success()); + + auto get_blob_property_outcome = client.get_blob_property(container_name, blob_name); + REQUIRE(get_blob_property_outcome.success()); + REQUIRE(get_blob_property_outcome.response().size == 4 * 1024 * 1024); + + std::stringbuf strbuf; + std::ostream os(&strbuf); + auto get_blob_outcome = client.download_blob_to_stream(container_name, blob_name, 0, 4 * 1024 * 1024, os).get(); + REQUIRE(get_blob_outcome.success()); + std::string all_0_string; + all_0_string.append(4 * 1024 * 1024, '\0'); + REQUIRE(strbuf.str() == all_0_string); + } + + SECTION("Clear 5MB page unsuccessfully") + { + auto clear_page_outcome = client.clear_page(container_name, blob_name, 0, 5 * 1024 * 1024).get(); + REQUIRE(!clear_page_outcome.success()); + } + + SECTION("Clear 1MB page to an 512 byte aligned offset successfully") + { + auto clear_page_outcome = client.clear_page(container_name, blob_name, 3 * 1024 * 1024, 1 * 1024 * 1024).get(); + REQUIRE(clear_page_outcome.success()); + + auto get_blob_property_outcome = client.get_blob_property(container_name, blob_name); + REQUIRE(get_blob_property_outcome.success()); + REQUIRE(get_blob_property_outcome.response().size == 4 * 1024 * 1024); + + std::stringbuf strbuf; + std::ostream os(&strbuf); + auto get_blob_outcome = client.download_blob_to_stream(container_name, blob_name, 0, 4 * 1024 * 1024, os).get(); + REQUIRE(get_blob_outcome.success()); + REQUIRE(strbuf.str() == iss.str().substr(0, 3 * 1024 * 1024).append(1024 * 1024, '\0')); + } + + SECTION("Clear 1MB page to an 512 not aligned offset unsuccessfully") + { + auto clear_page_outcome = client.clear_page(container_name, blob_name, 511, 1 * 1024 * 1024).get(); + REQUIRE(!clear_page_outcome.success()); + } + + SECTION("Clear 2MB page exceeds the size of the page blob unsuccessfully") + { + auto clear_page_outcome = client.clear_page(container_name, blob_name, 3 * 1024 * 1024, 2 * 1024 * 1024).get(); + REQUIRE(!clear_page_outcome.success()); + } + + client.delete_container(container_name); +} + +TEST_CASE("Get page ranges", "[page blob],[blob_service]") +{ + azure::storage_lite::blob_client client = as_test::base::test_blob_client(); + std::string container_name = as_test::create_random_container("", client); + std::string blob_name = as_test::get_random_string(20); + auto create_page_blob_outcome = client.create_page_blob(container_name, blob_name, 1024 * 20).get(); + REQUIRE(create_page_blob_outcome.success()); + for (unsigned i = 0; i < 10; ++i) + { + auto iss = as_test::get_istringstream_with_random_buffer(1024); + auto put_page_from_stream_outcome = client.put_page_from_stream(container_name, blob_name, i * 1024 * 2, iss.str().size(), iss).get(); + REQUIRE(put_page_from_stream_outcome.success()); + } + + + SECTION("Get all page ranges successfully") + { + auto get_page_range_outcome = client.get_page_ranges(container_name, blob_name, 0, 1024 * 20).get(); + REQUIRE(get_page_range_outcome.success()); + auto page_list = get_page_range_outcome.response().pagelist; + for (unsigned i = 0; i < 10; ++i) + { + REQUIRE(page_list[i].start == i * 2 * 1024); + REQUIRE(page_list[i].end == (i * 2 + 1) * 1024 - 1); + } + } + + SECTION("Get all page not 512 align successfully") + { + auto get_page_range_outcome = client.get_page_ranges(container_name, blob_name, 513, 4096).get(); + REQUIRE(get_page_range_outcome.success()); + auto page_list = get_page_range_outcome.response().pagelist; + REQUIRE(page_list[0].start == 512); + REQUIRE(page_list[0].end == 1023); + REQUIRE(page_list[1].start == 2048); + REQUIRE(page_list[1].end == 3071); + REQUIRE(page_list[2].start == 4096); + REQUIRE(page_list[2].end == 5119); + } + + SECTION("Get page ranges with offset larger than page blob size unsuccessfully") + { + auto get_page_range_outcome = client.get_page_ranges(container_name, blob_name, 1024 * 21, 1024 * 20).get(); + REQUIRE(!get_page_range_outcome.success()); + REQUIRE(get_page_range_outcome.error().code == "416"); + REQUIRE(get_page_range_outcome.error().code_name == "InvalidRange"); + } + + SECTION("Get page ranges with size larger than page blob size successfully") + { + auto get_page_range_outcome = client.get_page_ranges(container_name, blob_name, 0, 1024 * 25).get(); + REQUIRE(get_page_range_outcome.success()); + auto page_list = get_page_range_outcome.response().pagelist; + for (unsigned i = 0; i < 10; ++i) + { + REQUIRE(page_list[i].start == i * 2 * 1024); + REQUIRE(page_list[i].end == (i * 2 + 1) * 1024 - 1); + } + } + + client.delete_container(container_name); +} diff --git a/test/test_base.cpp b/test/test_base.cpp new file mode 100644 index 0000000..5127e1d --- /dev/null +++ b/test/test_base.cpp @@ -0,0 +1,165 @@ +#include "test_base.h" + +// tell Catch to provide a main() +#define CATCH_CONFIG_MAIN +#include "./catch2/catch.hpp" + +#include +#include +#include +#include + +namespace as_test { + std::string get_random_string(size_t size) { + static bool initialized = false; + if (!initialized) { + srand(time(NULL)); + initialized = true; + } + auto randchar = []() -> char + { + const char charset[] = + "0123456789abcdefghijklmnopqrstuvwxyz"; + const size_t max_index = (sizeof(charset) - 1); + return charset[rand() % max_index]; + }; + std::string str(size, 0); + std::generate_n(str.begin(), size, randchar); + return str; + } + + std::istringstream get_istringstream_with_random_buffer(size_t size) { + static bool initialized = false; + if (!initialized) { + srand(time(NULL)); + initialized = true; + } + auto randchar = []() -> char + { + const char charset[] = + "0123456789abcdefghijklmnopqrstuvwxyz"; + const size_t max_index = (sizeof(charset) - 1); + return charset[rand() % max_index]; + }; + std::string str(size, 0); + std::generate_n(str.begin(), size, randchar); + + std::istringstream ss; + ss.str(str); + return ss; + } + + std::string to_base64(const char* base, size_t length) + { + static const char* base64_enctbl = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + std::string result; + for (int offset = 0; length - offset >= 3; offset += 3) + { + const char* ptr = base + offset; + unsigned char idx0 = ptr[0] >> 2; + unsigned char idx1 = ((ptr[0] & 0x3) << 4) | ptr[1] >> 4; + unsigned char idx2 = ((ptr[1] & 0xF) << 2) | ptr[2] >> 6; + unsigned char idx3 = ptr[2] & 0x3F; + result.push_back(base64_enctbl[idx0]); + result.push_back(base64_enctbl[idx1]); + result.push_back(base64_enctbl[idx2]); + result.push_back(base64_enctbl[idx3]); + } + switch (length % 3) + { + case 1: + { + + const char* ptr = base + length - 1; + unsigned char idx0 = ptr[0] >> 2; + unsigned char idx1 = ((ptr[0] & 0x3) << 4); + result.push_back(base64_enctbl[idx0]); + result.push_back(base64_enctbl[idx1]); + result.push_back('='); + result.push_back('='); + break; + } + case 2: + { + + const char* ptr = base + length - 2; + unsigned char idx0 = ptr[0] >> 2; + unsigned char idx1 = ((ptr[0] & 0x3) << 4) | ptr[1] >> 4; + unsigned char idx2 = ((ptr[1] & 0xF) << 2); + result.push_back(base64_enctbl[idx0]); + result.push_back(base64_enctbl[idx1]); + result.push_back(base64_enctbl[idx2]); + result.push_back('='); + break; + } + } + return result; + } + + azure::storage_lite::blob_client& base::test_blob_client(int size) { + static std::unordered_map> bcs; + if (bcs[size] == NULL) + { + bcs[size] = std::make_shared(azure::storage_lite::blob_client(init_account(standard_storage_connection_string()), size)); + } + return *bcs[size]; + } + + const std::shared_ptr base::init_account(const std::string& connection_string) { + auto settings = parse_string_into_settings(connection_string); + auto credential = std::make_shared(azure::storage_lite::shared_key_credential(settings["AccountName"], settings["AccountKey"])); + bool use_https = true; + if (settings["DefaultEndpointsProtocol"] == "http") + { + use_https = false; + } + std::string blob_endpoint; + if (!settings["BlobEndpoint"].empty()) + { + blob_endpoint = settings["BlobEndpoint"]; + } + + return std::make_shared(azure::storage_lite::storage_account(settings["AccountName"], credential, use_https, blob_endpoint)); + } + + std::map base::parse_string_into_settings(const std::string& connection_string) + { + std::map settings; + std::vector splitted_string; + + // Split the connection string by ';' + { + std::istringstream iss(connection_string); + std::string s; + while (getline(iss, s, ';')) { + splitted_string.push_back(s); + } + } + + for (auto iter = splitted_string.cbegin(); iter != splitted_string.cend(); ++iter) + { + if (!iter->empty()) + { + auto equals = iter->find('='); + + std::string key = iter->substr(0, equals); + if (!key.empty()) + { + std::string value; + if (equals != std::string::npos) + { + value = iter->substr(equals + 1); + } + + settings.insert(std::make_pair(std::move(key), std::move(value))); + } + else + { + throw std::logic_error("The format of connection string cannot be recognized."); + } + } + } + + return settings; + } +} diff --git a/test/test_base.h b/test/test_base.h new file mode 100644 index 0000000..90e056e --- /dev/null +++ b/test/test_base.h @@ -0,0 +1,26 @@ +#pragma once + +#include "../include/blob/blob_client.h" + +#include + +namespace as_test { + + std::string get_random_string(size_t size); + std::istringstream get_istringstream_with_random_buffer(size_t size); + std::string to_base64(const char* base, size_t length); + + class base { + public: + static azure::storage_lite::blob_client& test_blob_client(int size = 1); + + static const std::string& standard_storage_connection_string() { + static std::string sscs = "DefaultEndpointsProtocol=https;"; + return sscs; + } + + private: + static const std::shared_ptr init_account(const std::string& connection_string); + static std::map parse_string_into_settings(const std::string& connection_string); + }; +} diff --git a/test/test_constants.cpp b/test/test_constants.cpp new file mode 100644 index 0000000..ca13abb --- /dev/null +++ b/test/test_constants.cpp @@ -0,0 +1,5 @@ +#include "test_constants.h" + +namespace as_test { + int MAX_PREFIX_SIZE = 20; +} \ No newline at end of file diff --git a/test/test_constants.h b/test/test_constants.h new file mode 100644 index 0000000..52bcacb --- /dev/null +++ b/test/test_constants.h @@ -0,0 +1,5 @@ +#pragma once + +namespace as_test { + extern int MAX_PREFIX_SIZE; +} \ No newline at end of file