#!/bin/bash
# AWS S3 bucket and object information
S3_BUCKET_NAME="your-bucket-name"
S3_OBJECT_KEY="backup/largefile.zip"
UPLOAD_ID="your-upload-id" # Replace with the actual upload ID
# List the parts uploaded for the given upload ID
parts_list=$(aws s3api list-parts --bucket "${S3_BUCKET_NAME}" --key "${S3_OBJECT_KEY}" --upload-id "${UPLOAD_ID}")
# Extract the ETags and part numbers from the parts list
ETAGS=($(echo "${parts_list}" | jq -r '.Parts[].ETag'))
PART_NUMBERS=($(echo "${parts_list}" | jq -r '.Parts[].PartNumber'))
# Create a JSON file containing the completed parts information
json_file=$(mktemp /tmp/completed_parts.XXXXXX.json)
echo '{"Parts": [' > "${json_file}"
for ((i = 0; i < ${#ETAGS[@]}; i++)); do
echo "{\"PartNumber\": ${PART_NUMBERS[$i]}, \"ETag\": \"${ETAGS[$i]}\"}," >> "${json_file}"
done
# Remove the trailing comma and close the JSON array
sed -i '$ s/,$//' "${json_file}"
echo ']}' >> "${json_file}"
# Complete the multipart upload
aws s3api complete-multipart-upload \
--bucket "${S3_BUCKET_NAME}" \
--key "${S3_OBJECT_KEY}" \
--upload-id "${UPLOAD_ID}" \
--multipart-upload "file://${json_file}"
# Clean up temporary JSON file
rm "${json_file}"
echo "Multi-part upload completed."
Multipart upload
#!/bin/bash
# AWS S3 bucket and object information
S3_BUCKET_NAME="your-bucket-name"
LOCAL_FILE_PATH="/path/to/your/largefile.zip"
S3_OBJECT_KEY="backup/largefile.zip"
# AWS CLI command to initiate a multi-part upload and retrieve the upload ID
upload_id=$(aws s3api create-multipart-upload --bucket "${S3_BUCKET_NAME}" --key "${S3_OBJECT_KEY}" --query 'UploadId' --output text)
if [ -z "${upload_id}" ]; then
echo "Failed to initiate multi-part upload."
exit 1
fi
echo "Multi-part upload initiated. Upload ID: ${upload_id}"
# Set up variables for part number and part size
part_number=1
part_size=5 # Specify the part size in megabytes (adjust as needed)
# Split the local file into parts and upload each part
split -b "${part_size}M" "${LOCAL_FILE_PATH}" "${LOCAL_FILE_PATH}.part"
for part_file in "${LOCAL_FILE_PATH}.part"*; do
echo "Uploading part ${part_number}..."
aws s3api upload-part --bucket "${S3_BUCKET_NAME}" --key "${S3_OBJECT_KEY}" --upload-id "${upload_id}" --part-number "${part_number}" --body "${part_file}" --output json
((part_number++))
done
# Complete the multi-part upload
aws s3api complete-multipart-upload --bucket "${S3_BUCKET_NAME}" --key "${S3_OBJECT_KEY}" --upload-id "${upload_id}" --multipart-upload '{"Parts": [{'$(for i in $(seq 1 $((part_number-1))); do echo -n "{\"PartNumber\":$i,\"ETag\":\"$(aws s3api list-parts --bucket "${S3_BUCKET_NAME}" --key "${S3_OBJECT_KEY}" --upload-id "${upload_id}" --query "Parts[?PartNumber==\`$i\`].ETag" --output text)\"},"; done)']}]}' --output json
# Clean up temporary part files
rm "${LOCAL_FILE_PATH}.part"*
echo "Multi-part upload complete."
My Gateway 