test.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from twspace_dl.api import API\n",
    "from twspace_dl.cookies import load_cookies\n",
    "from twspace_dl.twspace import Twspace\n",
    "from twspace_dl.twspace_dl import TwspaceDL\n",
    "import json\n",
    "import time\n",
    "import random\n",
    "import os\n",
    "import subprocess\n",
    "import openai\n",
    "from dotenv import load_dotenv\n",
    "\n",
    "load_dotenv()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def download_twitter_space_direct(space_url, cookie_file, output_format = \"/tmp/spaces/%(creator_name)s-%(creator_id)s-%(title)s-%(id)s\"):\n",
    "    API.init_apis(load_cookies(cookie_file))\n",
    "    twspace = Twspace.from_space_url(space_url)\n",
    "    \n",
    "    # Initialize TwspaceDL with a specific output format\n",
    "    twspace_dl = TwspaceDL(twspace, output_format)\n",
    "    \n",
    "    # This will now use the custom output format you provided\n",
    "    file_save_path = twspace_dl.filename + \".m4a\"\n",
    "    \n",
    "    try:\n",
    "        twspace_dl.download()\n",
    "        twspace_dl.embed_cover()\n",
    "    except KeyboardInterrupt:\n",
    "        print(\"Download Interrupted by user\")\n",
    "    finally:\n",
    "        twspace_dl.cleanup()\n",
    "    \n",
    "    if os.path.exists(file_save_path):\n",
    "        return file_save_path\n",
    "    else:\n",
    "        return None\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def monitor_twitter_spaces(user_ids, cookies_path, interval=10, variance=5):\n",
    "    while True:\n",
    "        for user_id in user_ids:\n",
    "            log_prefix = f\"[{time.strftime('%m/%d/%y %H:%M:%S')}] [tw_space@{user_id}] \"\n",
    "            \n",
    "            print(f\"{log_prefix} [VRB] Start trying with cookies...\")\n",
    "            space_url = f\"https://twitter.com/{user_id}\"\n",
    "            download_twitter_space_direct(space_url, cookies_path)\n",
    "\n",
    "            sleep_time = interval + random.randint(-variance, variance)\n",
    "            print(f\"{log_prefix} [VRB] Sleep {sleep_time} sec.\")\n",
    "            time.sleep(sleep_time)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[hls @ 0x12961aeb0] Changing ID3 metadata in HLS audio elementary stream is not implemented. Update your FFmpeg version to the newest one from Git. If the problem still occurs, it means that your file has a feature which has not been implemented.\n",
      "size=   19968kB time=00:29:15.17 bitrate=  93.2kbits/s speed=11.3x    \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Download Interrupted by user\n"
     ]
    }
   ],
   "source": [
    "download_twitter_space_direct(\"https://twitter.com/i/spaces/1mrxmyoNBmWxy?s=20\", \"cookies.txt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def chunk_file_if_needed(file_path, max_size_mb=25):\n",
    "    # Check the file size\n",
    "    file_size_mb = os.path.getsize(file_path) / (1024 * 1024)\n",
    "\n",
    "    if file_size_mb <= max_size_mb:\n",
    "        # If size is below threshold, return the original path in a list\n",
    "        return [file_path]\n",
    "    else:\n",
    "        # If file needs chunking\n",
    "        # Calculate segment time (as an estimate for now)\n",
    "        # This is a basic approach. Ideally, we might want to adjust based on file bitrate\n",
    "        duration = get_audio_duration(file_path)\n",
    "        estimated_segment_time = int(duration * (max_size_mb / file_size_mb))\n",
    "\n",
    "        # Create the directory to store segments\n",
    "        base_name = os.path.basename(file_path).rsplit('.', 1)[0]\n",
    "        segments_dir = f\"/tmp/spaces/{base_name}/segments\"\n",
    "        os.makedirs(segments_dir, exist_ok=True)\n",
    "\n",
    "        # Split the file into chunks\n",
    "        os.system(f\"ffmpeg -i {file_path} -f segment -segment_time {estimated_segment_time} -c copy {segments_dir}/segment%09d.mp3\")\n",
    "\n",
    "        # Return the list of chunk file paths\n",
    "        return [os.path.join(segments_dir, f) for f in os.listdir(segments_dir) if f.startswith(\"segment\")]\n",
    "\n",
    "def get_audio_duration(file_path):\n",
    "    # Using ffprobe to get the duration of the audio\n",
    "    cmd = f\"ffprobe -v error -show_entries format=duration -of default=noprint_wrappers=1:nokey=1 {file_path}\"\n",
    "    result = subprocess.check_output(cmd, shell=True)\n",
    "    return float(result)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "def transcribe_segments(segments, prompt):\n",
    "    \"\"\"\n",
    "    Transcribe the audio segments using OpenAI's Whisper API.\n",
    "\n",
    "    :param segments: List of paths to audio segment files.\n",
    "    :param prompt: A prompt to provide context for the transcription.\n",
    "    :return: A dictionary with segment paths as keys and their transcriptions as values.\n",
    "    \"\"\"\n",
    "\n",
    "    transcript = \"\"\n",
    "    for segment in segments:\n",
    "        with open(segment, \"rb\") as audio_file:\n",
    "            res = openai.Audio.transcribe(\"whisper-1\", audio_file)\n",
    "            transcript += str(res['text'])\n",
    "\n",
    "    return transcript\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain.prompts import PromptTemplate\n",
    "\n",
    "prompt_template = \"\"\"\n",
    "    You are an analytics professional at Lido Finance, a Liquid Staking protocol for Ethereum. You are given a transcript of Twitter Spaces in the crypto/web3 space that may or may not be related to Lido.\n",
    "    Given the transcript, you are writing structured notes in markdown format. Think of your notes as key takeaways, TLDRs, and executive summaries.\n",
    "\n",
    "    Your notes should be concise, detailed, and structured by topics. You know what information is especially important, and what is less important.\n",
    "    \n",
    "    Here is the transcript:\n",
    "    {text}\n",
    "    \n",
    "    YOUR NOTES:\"\"\"\n",
    "\n",
    "refine_template = \"\"\"\n",
    "    You are an analytics professional at Lido Finance, a Liquid Staking protocol for Ethereum. You are given a transcript of Twitter Spaces in the crypto/web3 space that may or may not be related to Lido.\n",
    "    Given the transcript, you are refining structured notes in markdown format. Think of your notes as key takeaways, TLDRs, and executive summaries.\n",
    "\n",
    "    Here is the existing note:\n",
    "    {existing_answer}\n",
    "    \n",
    "    We have the opportunity to refine the existing note (only if needed) with some more context below:\n",
    "    -----\n",
    "    {text}\n",
    "    -----\n",
    "    \n",
    "    Given the new context, refine the original note to make it more complete.\n",
    "    If the context isn't useful, return the original summary.\n",
    "\n",
    "    Your notes should be concise, detailed, and structured by topics. You know what information is especially important, and what is less important.\n",
    "\n",
    "    Use markdown formatting to its fullest to produce visually appealing, structured notes.\n",
    "    \"\"\"\n",
    "\n",
    "prompt = PromptTemplate.from_template(prompt_template)\n",
    "refine_prompt = PromptTemplate.from_template(refine_template)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain.schema import Document\n",
    "from langchain.chat_models import ChatOpenAI\n",
    "from langchain.chains.summarize import load_summarize_chain\n",
    "from langchain.text_splitter import RecursiveCharacterTextSplitter\n",
    "\n",
    "def summarize_transcript(transcript):\n",
    "    doc = Document(page_content=transcript)\n",
    "\n",
    "    # Summarize the document\n",
    "    text_splitter = RecursiveCharacterTextSplitter(chunk_size = 40000, chunk_overlap = 500, length_function = len, is_separator_regex=False)\n",
    "    docs = text_splitter.split_documents([doc])\n",
    "\n",
    "    chain_type = \"stuff\" if len(docs) == 1 else \"refine\"\n",
    "    chain = load_summarize_chain(ChatOpenAI(temperature=0, model_name=\"gpt-3.5-turbo-16k\"), chain_type=chain_type, question_prompt=prompt, refine_prompt=refine_prompt)\n",
    "\n",
    "    res = chain.run(docs)\n",
    "\n",
    "    return res"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[hls @ 0x15611e850] Changing ID3 metadata in HLS audio elementary stream is not implemented. Update your FFmpeg version to the newest one from Git. If the problem still occurs, it means that your file has a feature which has not been implemented.\n",
      "size=   18241kB time=00:26:09.79 bitrate=  95.2kbits/s speed=62.4x    \n"
     ]
    }
   ],
   "source": [
    "transcript_location = download_twitter_space_direct(\"https://twitter.com/i/spaces/1OdKrjPEbLzKX?s=20\", \"cookies.txt\")\n",
    "chunks = chunk_file_if_needed(transcript_location)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "transcript = transcribe_segments(chunks, \"Twitter Space about Crypto, Web3, Liquid Staking, and Lido Finance\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "summary = summarize_transcript(transcript)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# Twitter Spaces Transcript Notes\n",
      "\n",
      "## Introduction\n",
      "- The Twitter Spaces session is co-hosted by Maverick Protocol and Lido Finance.\n",
      "- The session aims to discuss building in the current market conditions and the connection between Maverick and Lido.\n",
      "- The session features Ada Woo, a core contributor at Maverick, and Seraphim, a DeFi expansionist at Lido.\n",
      "\n",
      "## Maverick Protocol Introduction\n",
      "- Maverick Protocol is a DeFi infrastructure provider focused on increasing industry efficiency.\n",
      "- Maverick AMM brings higher capital efficiency to DeFi, resulting in lower slippage and more value stability.\n",
      "- Maverick DEX has ranked as a top three DEX on all chains by seven-day volume, supporting over 50% of the trading volume on L2 scaling solutions.\n",
      "- Maverick Boosted Positions maximize incentive efficiency for protocols looking to bootstrap liquidity or defend the peg.\n",
      "- Maverick AMM offers liquidity movement modes, including bullish, bearish, and both, allowing LPs to deploy strategies based on their market beliefs.\n",
      "\n",
      "## Lido Finance Introduction\n",
      "- Lido is the largest liquid staking protocol in the DeFi space.\n",
      "- Lido allows users to stake their ETH and receive a liquid staking token (wstETH) in return, preserving the utility of their staking position.\n",
      "- Lido has seen increased adoption during the bear market, as more ETH holders are staking their assets.\n",
      "- Lido plans to decentralize its node operator set from 32 to 3,500 node operators, allowing anyone to become a node operator and contribute to the network's decentralization.\n",
      "\n",
      "## Capital Efficiency and Liquidity\n",
      "- Maverick AMM supports the highest capital efficiency ratio for wstETH.\n",
      "- Capital efficiency refers to the volume divided by total locked value (TVL), indicating how much trading volume is supported with a given TVL.\n",
      "- Higher capital efficiency results in lower slippage, more stable value, and increased fees for liquidity providers.\n",
      "- Maverick achieves high capital efficiency through its dynamic distribution AMM, which allows LPs to customize liquidity distribution and movement modes.\n",
      "- Liquidity movement modes include bullish, bearish, and both, enabling LPs to support volume while the market goes sideways.\n",
      "- Maverick Boosted Positions allow token builders and communities to target liquidity incentives towards specific areas within a pool, maximizing incentivization efficiency.\n",
      "\n",
      "## Building in a Bear Market\n",
      "- Lido has experienced growth during the bear market, with increased staking rates and adoption.\n",
      "- Lido benefits from the liquidity and ability to sell large amounts instantly, attracting trading volume.\n",
      "- Maverick can help projects facing challenges in a bear market by providing high capital efficiency and boosted positions for liquidity shaping.\n",
      "- Building in a bear market allows projects to focus on product development and stand out with less market noise and distractions.\n",
      "\n",
      "## Opportunities in a Pre-Bull Market\n",
      "- Lido sees opportunities in a pre-bull market as interest rates in traditional markets decrease.\n",
      "- Lower interest rates may lead to more ETH holders staking their assets, increasing demand for liquid staking tokens like wstETH.\n",
      "- Maverick users can benefit from more incentive opportunities for liquidity providers and participate in the Maverick ecosystem incentive program.\n",
      "- Lido plans to further decentralize its node operator set, allowing more participants to become node operators and contribute to the network's security and decentralization.\n",
      "\n",
      "## Maverick Ecosystem Incentive Program\n",
      "- Maverick has launched its season one of the ecosystem incentive program.\n",
      "- Besides liquidity providers, new token builders are also welcome to participate in the leaderboard.\n",
      "- The leaderboard can be checked on the app.map.xyz/leaderboard.\n",
      "- The program aims to provide more incentive opportunities for liquidity providers and token projects to grow on top of the liquidity shaping tool, Boosted Positions.\n",
      "\n",
      "## Maverick Phase Three and VMF Model Launch\n",
      "- Maverick is getting closer to launching its phase three, which is a full VMF model launch.\n",
      "- The VMF model will be based on Maverick Boosted Positions and will provide new tokens or token projects with even more efficient liquidity shaping.\n",
      "- The launch of the VMF model will bring exciting opportunities for liquidity providers to earn more incentives.\n",
      "\n",
      "## Conclusion\n",
      "- Lido is working on decentralizing its node operator set, allowing more participants to become node operators.\n",
      "- Maverick offers high capital efficiency and boosted positions for liquidity shaping.\n",
      "- Building in a bear market allows projects to focus on development and stand out with less market noise.\n",
      "- Opportunities in a pre-bull market include increased demand for liquid staking tokens and more incentive opportunities for liquidity providers.\n",
      "- Both Lido and Maverick are well-positioned to thrive in the current market conditions.\n"
     ]
    }
   ],
   "source": [
    "print(summary)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": ".venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.5"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}