🧑‍💻Programming

How to programmatically work with AWS S3

Commands

List Buckets

CLI

aws s3 ls

Python3

import boto3

# Create a session using your AWS credentials
session = boto3.Session(
    aws_access_key_id='your_access_key_id',
    aws_secret_access_key='your_secret_access_key',
    region_name='your_preferred_region'
)

# Create an S3 client
s3 = session.client('s3')

# List all buckets
response = s3.list_buckets()

# Print bucket names
print("Existing buckets:")
for bucket in response['Buckets']:
    print(f"  {bucket['Name']}")

Copy Bucket File(s) Locally

CLI

aws s3 cp s3://my-bucket/my-object /local/dir

aws s3 cp s3://my-bucket/my-object /local/dir --recursive

Python3

Code to copy an object from S3.

import boto3

# Initialize a session using Amazon S3
s3 = boto3.client('s3')

# Define the bucket name and object key
bucket_name = 'my-bucket'
object_key = 'my-object'
local_file_path = '/local/dir/my-object'  # Adjust the path and filename as needed

# Download the object from S3
try:
    s3.download_file(bucket_name, object_key, local_file_path)
    print(f'Successfully downloaded {object_key} from bucket {bucket_name} to {local_file_path}')
except Exception as e:
    print(f'Error downloading file: {e}')

Code to copy multiple objects from S3.

import os
import boto3

# Initialize a session using Amazon S3
s3 = boto3.client('s3')

# Define the bucket name and prefix (folder) if any
bucket_name = 'my-bucket'
prefix = 'my-folder/'  # If you want to download everything, set prefix to ''

# Local directory to save the downloaded files
local_dir = '/local/dir'

# Ensure the local directory exists
if not os.path.exists(local_dir):
    os.makedirs(local_dir)

# List objects in the specified bucket and prefix
paginator = s3.get_paginator('list_objects_v2')
pages = paginator.paginate(Bucket=bucket_name, Prefix=prefix)

for page in pages:
    if 'Contents' in page:
        for obj in page['Contents']:
            key = obj['Key']
            # Construct local file path
            local_file_path = os.path.join(local_dir, key[len(prefix):])
            local_file_dir = os.path.dirname(local_file_path)

            # Ensure the local directory exists
            if not os.path.exists(local_file_dir):
                os.makedirs(local_file_dir)

            # Download the file
            try:
                s3.download_file(bucket_name, key, local_file_path)
                print(f'Successfully downloaded {key} to {local_file_path}')
            except Exception as e:
                print(f'Error downloading {key}: {e}')

---

Resources

s3 — AWS CLI 2.17.15 Command Reference

Boto3 documentation — Boto3 Docs 1.24.22 documentation