Boto_AWS.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Based on realpython.com tutorial on Boto3\n",
    "import boto3\n",
    "import uuid"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "First step is to create a directory ~/.aws in your home directory using mkdir command. Inside that directory add two files called credentials and config with the following entries:\n",
    "credentials - >\n",
    "\n",
    "[default]\n",
    "\n",
    "aws_access_key_id = your id from aws user \n",
    "\n",
    "aws_secret_access_key = your secrets key\n",
    "\n",
    "config - >\n",
    "\n",
    "[default]\n",
    "\n",
    "region = eu-west-3"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Create a S3 bucket using the credential and config file (boto3 automatically looks for those files and only creates the bucket on AWS if that folder and files inside it are present)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mariproject71a3efa5-d4da-44d5-8236-2d937a6913dc\n",
      "mariproject71a3efa5-d4da-44d5-8236-2d937a6913dc s3.Bucket(name='mariproject71a3efa5-d4da-44d5-8236-2d937a6913dc')\n"
     ]
    }
   ],
   "source": [
    "s3_client = boto3.client('s3')\n",
    "s3_resource = boto3.resource('s3')\n",
    "\n",
    "def create_bucket_name(bucket_prefix):\n",
    "    \n",
    "    return ''.join([bucket_prefix,str(uuid.uuid4())])\n",
    "\n",
    "def create_bucket(bucket_prefix, s3_connection):\n",
    "    session = boto3.session.Session()\n",
    "    current_region = session.region_name\n",
    "    bucket_name = create_bucket_name(bucket_prefix)\n",
    "    print(bucket_name)\n",
    "    bucket_response = s3_connection.create_bucket( Bucket = bucket_name, CreateBucketConfiguration = {\n",
    "                 'LocationConstraint': current_region\n",
    "    })\n",
    "    return bucket_name, bucket_response    \n",
    "bucket_name, response = create_bucket('mariproject', s3_resource)\n",
    "print(bucket_name, response)    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Upload file to a bucket"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "s3.Object(bucket_name='mariproject71a3efa5-d4da-44d5-8236-2d937a6913dc', key='./Boto_AWS.ipynb')"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bucket = s3_resource.Bucket(bucket_name)\n",
    "file_name = './Boto_AWS.ipynb'\n",
    "bucket_object = bucket.Object(file_name)\n",
    "bucket_object"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "bucket_object.upload_file(file_name)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Download file from bucket"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "bucket_object.download_file(f'./tmp/test/{file_name}')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Move/Copy objects between buckets "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "def copy_to_bucket(s3_connection, from_bucket, to_bucket, file_name):\n",
    "    copy_source = {\n",
    "        'Bucket': from_bucket,\n",
    "        'Key': file_name\n",
    "    }\n",
    "    s3_connection.Object(to_bucket, file_name).copy(copy_source)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "second_bucket_name = 'marioneat'\n",
    "copy_to_bucket(s3_resource, bucket_name, second_bucket_name, file_name)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Iterate over buckets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Bucket Name marioneat\n",
      "Bucket object s3.Bucket(name='marioneat')\n",
      "key ./Boto_AWS.ipynb\n",
      "key Mari_Models/Oneat/Cellsplitcategoriesxenopus.json\n",
      "key Mari_Models/Oneat/Cellsplitcordxenopus.json\n",
      "key Mari_Models/Oneat/Cellsplitdetectorxenopuspurelstm_d29_f8.h5.png\n",
      "key Mari_Models/Oneat/Cellsplitdetectorxenopuspurelstm_d29_f8_Parameter.json\n",
      "key Mari_Models/Oneat/Cellsplitdetectorxenopusvolumecnn_d11_f16.h5.png\n",
      "key Mari_Models/Oneat/Cellsplitdetectorxenopusvolumecnn_d11_f16_Parameter.json\n",
      "key Mari_Models/Oneat/Cellsplitdetectorxenopusvolumecnn_d11_f64.h5.png\n",
      "key Mari_Models/Oneat/Cellsplitdetectorxenopusvolumecnn_d11_f64_Parameter.json\n",
      "key Mari_Models/Oneat/Cellsplitdetectorxenopusvolumecnn_d29_f16.h5\n",
      "key Mari_Models/Oneat/Cellsplitdetectorxenopusvolumecnn_d29_f16.h5.png\n",
      "key Mari_Models/Oneat/Cellsplitdetectorxenopusvolumecnn_d29_f16_Parameter.json\n",
      "key Mari_Models/Oneat/Cellsplitdetectorxenopusvolumecnn_d29_f32.h5\n",
      "key Mari_Models/Oneat/Cellsplitdetectorxenopusvolumecnn_d29_f32.h5.png\n",
      "key Mari_Models/Oneat/Cellsplitdetectorxenopusvolumecnn_d29_f32_Parameter.json\n",
      "key Mari_Models/Oneat/Cellsplitdetectorxenopusvolumecnn_d29_f64.h5\n",
      "key Mari_Models/Oneat/Cellsplitdetectorxenopusvolumecnn_d29_f64.h5.png\n",
      "key Mari_Models/Oneat/Cellsplitdetectorxenopusvolumecnn_d29_f64_Parameter.json\n",
      "key Mari_Models/Oneat/Cellsplitdetectorxenopusvolumecnn_d38_f16.h5.png\n",
      "key Mari_Models/Oneat/Cellsplitdetectorxenopusvolumecnn_d38_f16_Parameter.json\n",
      "key Mari_Models/Oneat/Cellsplitdiamondcategoriesxenopus.json\n",
      "key Mari_Models/Oneat/Cellsplitdiamondcordxenopus.json\n",
      "key Xenopus_oneat_training_m1p1_diamond.npz\n",
      "key Xenopus_oneat_training_m1p1_diamondval.npz\n",
      "key train_xenopus_oneat_diamond.py\n",
      "Bucket Name mariproject71a3efa5-d4da-44d5-8236-2d937a6913dc\n",
      "Bucket object s3.Bucket(name='mariproject71a3efa5-d4da-44d5-8236-2d937a6913dc')\n",
      "key ./Boto_AWS.ipynb\n",
      "Bucket Name mariprojectc1634c3b-cfba-4144-a1f5-5960413fe310\n",
      "Bucket object s3.Bucket(name='mariprojectc1634c3b-cfba-4144-a1f5-5960413fe310')\n",
      "key ./Boto_AWS.ipynb\n",
      "Bucket Name sagemaker-studio-036934985750-oxkpm2dh66n\n",
      "Bucket object s3.Bucket(name='sagemaker-studio-036934985750-oxkpm2dh66n')\n",
      "Bucket Name sagemaker-studio-1ion02gbqkc\n",
      "Bucket object s3.Bucket(name='sagemaker-studio-1ion02gbqkc')\n",
      "Bucket Name sagemaker-studio-khc9p4pvpjc\n",
      "Bucket object s3.Bucket(name='sagemaker-studio-khc9p4pvpjc')\n"
     ]
    }
   ],
   "source": [
    "all_buckets = s3_resource.buckets.all()\n",
    "for bucket in all_buckets:\n",
    "    print('Bucket Name', bucket.name)\n",
    "    bucket_object_current = s3_resource.Bucket(name = bucket.name)\n",
    "    print('Bucket object',bucket_object_current)\n",
    "    for object_current in bucket_object_current.objects.all():\n",
    "        print('key', object_current.key)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Delete file from the first bucket"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'ResponseMetadata': {'RequestId': '8MHE940AP45ATJ3J',\n",
       "  'HostId': 'dMWes+6cyiMHrlQUvUL54j0t6uZTYKeiy8qbiArWL4DQCXZyXwDeQVCbdqc5yuwSTzlfCXANnoqcH24RpdOssw==',\n",
       "  'HTTPStatusCode': 204,\n",
       "  'HTTPHeaders': {'x-amz-id-2': 'dMWes+6cyiMHrlQUvUL54j0t6uZTYKeiy8qbiArWL4DQCXZyXwDeQVCbdqc5yuwSTzlfCXANnoqcH24RpdOssw==',\n",
       "   'x-amz-request-id': '8MHE940AP45ATJ3J',\n",
       "   'date': 'Thu, 29 Sep 2022 21:28:38 GMT',\n",
       "   'server': 'AmazonS3'},\n",
       "  'RetryAttempts': 0}}"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bucket_object.delete()\n",
    "#Sucessful request has 204 code"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.9.0 ('naparienv')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.0"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "e140276ae15c1be7b597ddfede76c3757c35dc6a4240f18994000df39384733e"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}