cubefs

1
# Copyright 2020 The CubeFS Authors.
2
#
3
# Licensed under the Apache License, Version 2.0 (the "License");
4
# you may not use this file except in compliance with the License.
5
# You may obtain a copy of the License at
6
#
7
#     http://www.apache.org/licenses/LICENSE-2.0
8
#
9
# Unless required by applicable law or agreed to in writing, software
10
# distributed under the License is distributed on an "AS IS" BASIS,
11
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
12
# implied. See the License for the specific language governing
13
# permissions and limitations under the License.
14

15
# -*- coding: utf-8 -*-
16
import time
17

18
from env import BUCKET
19
from base import S3TestCase, get_env_s3_client
20

21
SOURCE_KEY = "copyTest/key/sourceKey.txt"
22
TARGET_KEY = "copyTest/key/targetKey.txt"
23

24
SOURCE_KEY_DIR = "copyTest/dir/targetDir/"
25
TARGET_KEY_DIR = "copyTest/dir/sourceDir/"
26

27
SOURCE_KEY_WITH_META = "copyTest/key/withMeta/sourceKey.txt"
28
TARGET_KEY_WITH_META = "copyTest/key/withMeta/targetKey.txt"
29

30
SOURCE_KEY_RESET_META = "copyTest/key/reset/sourceKey.txt"
31
TARGET_KEY_RESET_META = "copyTest/key/reset/targetKey.txt"
32

33
SOURCE_KEY_MODIFY_META = "copyTest/key/modify/sourceKey.txt"
34

35
META_DATE_KEY_1 = "sourceKeyMetaKey1"
36
META_DATE_KEY_2 = "sourceKeyMetaKey2"
37
META_DATE_VALUE_1 = "sourceKeyMetaValue1"
38
META_DATE_VALUE_2 = "sourceKeyMetaValue2"
39
META_DATE_VALUE_1_MODIFIED = "sourceKeyMetaValue1Modified"
40

41

42
class CopyObjectTest(S3TestCase):
43
    s3 = get_env_s3_client()
44

45
    def __init__(self, case):
46
        super(CopyObjectTest, self).__init__(case)
47

48
    @classmethod
49
    def setUpClass(cls):
50
        """
51
        Create test data, such as putting object of source keys.
52
        :return:
53
        """
54
        cls.clear_data()
55
        # create source object info
56
        cls.create_key(key=SOURCE_KEY, content=b'copyTest source key content')
57
        cls.create_key(key=SOURCE_KEY_DIR, content='')
58
        cls.create_key(key=SOURCE_KEY_WITH_META, content=b'copyTest source key with meta data', mete_data=True)
59
        cls.create_key(key=SOURCE_KEY_RESET_META, content=b'copyTest source key for used reset meta data')
60

61
    @classmethod
62
    def tearDownClass(cls):
63
        """
64
        Clean temp data, include initialized test data, create middle temp data and result data.
65
        :return:
66
        """
67
        cls.clear_data()
68

69
    @classmethod
70
    def create_key(cls, key, content, mete_data=False):
71
        """
72
        :return:
73
        """
74
        if mete_data:
75
            metadata = {META_DATE_KEY_1: META_DATE_VALUE_1, META_DATE_KEY_2: META_DATE_VALUE_2}
76
            cls.s3.put_object(Bucket=BUCKET, Key=key, Body=content, Metadata=metadata)
77
        else:
78
            cls.s3.put_object(Bucket=BUCKET, Key=key, Body=content)
79

80
    @classmethod
81
    def delete_key(cls, key):
82
        """
83
        :return:
84
        """
85
        cls.s3.delete_object(Bucket=BUCKET, Key=key)
86

87
    def __copy_object(self, s_bucket, s_key, t_bucket, t_key, is_dir=False, contain_mete_data=False):
88
        # sleep two seconds, otherwise target key last modified is same with the source
89
        time.sleep(2)
90
        copy_source = {'Bucket': s_bucket, 'Key': s_key}
91
        self.s3.copy_object(CopySource=copy_source, Bucket=t_bucket, Key=t_key)
92
        source_response = self.s3.head_object(Bucket=s_bucket, Key=s_key)
93
        target_response = self.s3.head_object(Bucket=t_bucket, Key=t_key)
94
        self.assertNotEqual(target_response["ETag"], "")
95
        self.assertEqual(target_response["ETag"], source_response["ETag"])
96
        self.assertEqual(target_response["ContentLength"], source_response["ContentLength"])
97
        self.assertGreater(target_response["LastModified"], source_response["LastModified"])
98
        if is_dir:
99
            self.assertEqual(target_response["ContentLength"], 0)
100
        if contain_mete_data:
101
            target_meta_data = target_response["Metadata"]
102
            # target object must have metadata
103
            # The response returned metadata key we specified is lower,
104
            # so when using this metadata, we need to transfer metadata key to lower
105
            self.assertIsNotNone(target_meta_data)
106
            print("target_meta_data.keys():", target_meta_data.keys())
107
            self.assertTrue(META_DATE_KEY_1.lower().capitalize() in target_meta_data.keys())
108
            self.assertTrue(META_DATE_KEY_2.lower().capitalize() in target_meta_data.keys())
109
            self.assertEqual(target_meta_data[META_DATE_KEY_1.lower().capitalize()], META_DATE_VALUE_1)
110
            self.assertEqual(target_meta_data[META_DATE_KEY_2.lower().capitalize()], META_DATE_VALUE_2)
111

112
    @classmethod
113
    def clear_data(cls):
114
        cls.delete_key(key=SOURCE_KEY)
115
        cls.delete_key(key=TARGET_KEY)
116
        cls.delete_key(key=SOURCE_KEY_DIR)
117
        cls.delete_key(key=TARGET_KEY_DIR)
118
        cls.delete_key(key=SOURCE_KEY_WITH_META)
119
        cls.delete_key(key=TARGET_KEY_WITH_META)
120
        cls.delete_key(key=SOURCE_KEY_RESET_META)
121
        cls.delete_key(key=TARGET_KEY_RESET_META)
122
        cls.delete_key(key=SOURCE_KEY_MODIFY_META)
123

124
    def test_copy_common_key(self):
125
        """
126
        Copy common file, using default value.
127
        :return:
128
        """
129
        self.__copy_object(s_bucket=BUCKET,
130
                           s_key=SOURCE_KEY,
131
                           t_bucket=BUCKET,
132
                           t_key=TARGET_KEY)
133

134
    def test_copy_dir(self):
135
        """
136
        Copy directory, the source key is a directory(object content is empty, and key path has suffix '/').
137
        The target key is a directory too.
138
        :return:
139
        """
140
        self.__copy_object(s_bucket=BUCKET,
141
                           s_key=SOURCE_KEY_DIR,
142
                           t_bucket=BUCKET,
143
                           t_key=TARGET_KEY_DIR,
144
                           is_dir=True)
145

146
    def test_copy_metadata(self):
147
        """
148
        Copy source object metadata.
149
        If the source object has self_defined metadata, target object has its metadata too in default.
150
        :return:
151
        """
152
        self.__copy_object(s_bucket=BUCKET,
153
                           s_key=SOURCE_KEY_WITH_META,
154
                           t_bucket=BUCKET,
155
                           t_key=TARGET_KEY_WITH_META,
156
                           contain_mete_data=True)
157

158
    def test_copy_reset_metadata(self):
159
        """
160
        Reset target object metadata, no matter whether source object has self_defined metadata.
161
        :return:
162
        """
163
        source_bucket = BUCKET
164
        target_bucket = BUCKET
165
        source_key = SOURCE_KEY_RESET_META
166
        target_key = TARGET_KEY_RESET_META
167
        # sleep one second, otherwise target key last modified is same with the source
168
        time.sleep(1)
169
        copy_source = {'Bucket': source_bucket, 'Key': source_key}
170
        metadata = {META_DATE_KEY_1: META_DATE_VALUE_1, META_DATE_KEY_2: META_DATE_VALUE_2}
171
        self.s3.copy_object(CopySource=copy_source,
172
                            Bucket=target_bucket,
173
                            Key=target_key,
174
                            MetadataDirective="REPLACE",
175
                            Metadata=metadata)
176

177
        source_response = self.s3.head_object(Bucket=source_bucket, Key=source_key)
178
        target_response = self.s3.head_object(Bucket=target_bucket, Key=target_key)
179
        # compare basic info
180
        self.assertNotEqual(target_response["ETag"], "")
181
        self.assertEqual(target_response["ETag"], source_response["ETag"])
182
        self.assertEqual(target_response["ContentLength"], source_response["ContentLength"])
183
        self.assertGreater(target_response["LastModified"], source_response["LastModified"])
184
        # compare metadata
185
        # source key not contain metadata
186
        # target key not contain metadata
187
        source_metadata = source_response["Metadata"]
188
        target_metadata = target_response["Metadata"]
189
        self.assertEqual(len(source_metadata), 0)
190
        self.assertEqual(len(target_metadata), 2)
191
        self.assertTrue(META_DATE_KEY_1.lower().capitalize() in target_metadata.keys())
192
        self.assertTrue(META_DATE_KEY_2.lower().capitalize() in target_metadata.keys())
193
        self.assertEqual(target_metadata[META_DATE_KEY_1.lower().capitalize()], META_DATE_VALUE_1)
194
        self.assertEqual(target_metadata[META_DATE_KEY_2.lower().capitalize()], META_DATE_VALUE_2)
195

196
    def test_copy_modify_metadata(self):
197
        """
198
        Modify a object's metadata via specifying the target key has same path with source object,
199
        and specify new metadata value.
200
        :return:
201
        """
202
        metadata = {META_DATE_KEY_1: META_DATE_VALUE_1}
203
        content = "b'copyTest source key for used modify meta data'"
204
        self.s3.put_object(Bucket=BUCKET, Key=SOURCE_KEY_MODIFY_META, Body=content, Metadata=metadata)
205
        copy_source = {'Bucket': BUCKET, 'Key': SOURCE_KEY_MODIFY_META}
206
        metadata = {META_DATE_KEY_1: META_DATE_VALUE_1_MODIFIED}
207
        self.s3.copy_object(CopySource=copy_source,
208
                            Bucket=BUCKET,
209
                            Key=SOURCE_KEY_MODIFY_META,
210
                            MetadataDirective="REPLACE",
211
                            Metadata=metadata)
212
        response = self.s3.head_object(Bucket=BUCKET, Key=SOURCE_KEY_MODIFY_META)
213
        # target key not contain metadata
214
        metadata = response["Metadata"]
215
        self.assertEqual(len(metadata), 1)
216
        self.assertTrue(META_DATE_KEY_1.lower().capitalize() in metadata.keys())
217
        self.assertEqual(metadata[META_DATE_KEY_1.lower().capitalize()], META_DATE_VALUE_1_MODIFIED)
218