From 3e0a3fd344dedf86debf519c47282101e03ef2e3 Mon Sep 17 00:00:00 2001 From: zjregee <89204657+zjregee@users.noreply.github.com> Date: Sun, 3 Mar 2024 18:48:28 +0800 Subject: [PATCH] feat(services/swift): add ceph test setup for swift (#4307) * add ceph test setup * fix typos * modify ListOpResponse * update code format * fix wrong test and add more context --- .../swift/ceph_rados_swift/action.yml | 49 +++++++++++++++++++ core/src/services/swift/core.rs | 6 +-- core/src/services/swift/lister.rs | 13 +++-- fixtures/swift/docker-compose-ceph-rados.yml | 44 +++++++++++++++++ 4 files changed, 105 insertions(+), 7 deletions(-) create mode 100644 .github/services/swift/ceph_rados_swift/action.yml create mode 100644 fixtures/swift/docker-compose-ceph-rados.yml diff --git a/.github/services/swift/ceph_rados_swift/action.yml b/.github/services/swift/ceph_rados_swift/action.yml new file mode 100644 index 000000000000..a4665212229d --- /dev/null +++ b/.github/services/swift/ceph_rados_swift/action.yml @@ -0,0 +1,49 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: ceph_rados_swift +description: 'Behavior test for CEPH OBJECT GATEWAY Swift.' + +runs: + using: "composite" + steps: + - name: Setup Ceph Rados Server + shell: bash + working-directory: fixtures/swift + run: docker compose -f docker-compose-ceph-rados.yml up -d --wait + + - name: Create environment variables and setup test container + shell: bash + working-directory: fixtures/swift + run: | + docker exec ceph-demo radosgw-admin user create \ + --subuser="test_user:test_swift_user" --uid="test_user" --display-name="test_user" \ + --key-type=swift --secret="password" --access=full + response=$(curl -i -H "X-Auth-User: test_user:test_swift_user" -H "X-Auth-Key: password" http://127.0.0.1:8080/auth/v1) + token=$(echo "$response" | grep X-Auth-Token | head -n1 | awk '{print $2}' | tr -d '[:space:]') + endpoint=$(echo "$response" | grep X-Storage-Url | head -n1 | awk '{print $2}' | tr -d '[:space:]') + curl --location --request PUT "${endpoint}/testing" --header "X-Auth-Token: $token" + echo "OPENDAL_SWIFT_TOKEN=${token}" >> $GITHUB_ENV + echo "OPENDAL_SWIFT_ENDPOINT=${endpoint}" >> $GITHUB_ENV + + - name: Set environment variables + shell: bash + run: | + cat << EOF >> $GITHUB_ENV + OPENDAL_SWIFT_CONTAINER=testing + OPENDAL_SWIFT_ROOT=/ + EOF diff --git a/core/src/services/swift/core.rs b/core/src/services/swift/core.rs index 05249f874e42..1eb0f3d2a94e 100644 --- a/core/src/services/swift/core.rs +++ b/core/src/services/swift/core.rs @@ -229,8 +229,8 @@ pub enum ListOpResponse { bytes: u64, hash: String, name: String, - content_type: String, last_modified: String, + content_type: Option, }, } @@ -270,10 +270,8 @@ mod tests { bytes: 147, hash: "5e6b5b70b0426b1cc1968003e1afa5ad".to_string(), name: "test.txt".to_string(), - content_type: - "multipart/form-data;boundary=------------------------25004a866ee9c0cb" - .to_string(), last_modified: "2023-11-01T03:00:23.147480".to_string(), + content_type: Some("text/plain".to_string()), } ); diff --git a/core/src/services/swift/lister.rs b/core/src/services/swift/lister.rs index 464f36a9c2f7..ee90a1753777 100644 --- a/core/src/services/swift/lister.rs +++ b/core/src/services/swift/lister.rs @@ -95,10 +95,17 @@ impl oio::PageList for SwiftLister { meta.set_content_length(bytes); meta.set_content_md5(hash.as_str()); - // we'll change "2023-10-28T19:18:11.682610" to "2023-10-28T19:18:11.682610Z" - last_modified.push('Z'); + // OpenStack Swift returns time without 'Z' at the end, + // which causes an error in parse_datetime_from_rfc3339. + // we'll change "2023-10-28T19:18:11.682610" to "2023-10-28T19:18:11.682610Z". + if !last_modified.ends_with('Z') { + last_modified.push('Z'); + } meta.set_last_modified(parse_datetime_from_rfc3339(last_modified.as_str())?); - meta.set_content_type(content_type.as_str()); + + if let Some(content_type) = content_type { + meta.set_content_type(content_type.as_str()); + } oio::Entry::with(name, meta) } diff --git a/fixtures/swift/docker-compose-ceph-rados.yml b/fixtures/swift/docker-compose-ceph-rados.yml new file mode 100644 index 000000000000..c43ae6b2d427 --- /dev/null +++ b/fixtures/swift/docker-compose-ceph-rados.yml @@ -0,0 +1,44 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +version: "3.8" + +services: + ceph-demo: + image: quay.io/ceph/demo + container_name: ceph-demo + environment: + - MON_IP=127.0.0.1 + - CEPH_PUBLIC_NETWORK=0.0.0.0/0 + - DEMO_DAEMONS=osd,mds,rgw + - CEPH_DEMO_UID=demo + - CEPH_DEMO_ACCESS_KEY=demo + - CEPH_DEMO_SECRET_KEY=demo + ports: + - "8080:8080" + - "5000:5000" + - "6789:6789" + volumes: + - ceph-vol:/var/lib/ceph/ + network_mode: "host" + healthcheck: + test: [ "CMD", "radosgw-admin", "user", "list" ] + interval: 6s + retries: 100 + +volumes: + ceph-vol: