fileID_check.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "from matplotlib.pyplot import figure\n",
    "from scipy.optimize import curve_fit\n",
    "from scipy.signal import find_peaks\n",
    "from itertools import chain\n",
    "from sys import platform\n",
    "from copy import deepcopy as dc\n",
    "\n",
    "vims_wave = np.loadtxt('vims_wave.txt')\n",
    "\n",
    "new_array = ['_0405','_0607','_0809','_1011','_1213','_1415','_1617']\n",
    "\n",
    "if platform == 'darwin' or platform == 'win32':\n",
    "    for number in new_array:\n",
    "        new_string = 'df'+number+' = pd.read_pickle(\"c_data/data'+number+'.pkl\")'\n",
    "        exec(new_string)\n",
    "else:\n",
    "    df_list = []\n",
    "    for number in new_array:\n",
    "        new_string = 'df'+number+' = pd.read_csv(\"c_data/data'+number+'.csv\")'\n",
    "        exec(new_string)\n",
    "        df = 'df'+number\n",
    "        new_df = locals()[df]\n",
    "        df_list.append(new_df)\n",
    "    for df in df_list:\n",
    "        for i in range(len(df['spectrum'])):\n",
    "            df['spectrum'][i] = np.array([float(x) for x in df['spectrum'][i][1:-1].split(',')])\n",
    "\n",
    "def powerlaw(x,a,b):\n",
    "    return a*np.power(x,b)\n",
    "\n",
    "def gaussian(x,a,mu,sigma):\n",
    "    return a*np.exp(-(x-mu)**2/(2*sigma**2))\n",
    "\n",
    "def flatten(some_list):\n",
    "    flat_list = []\n",
    "    for element in some_list:\n",
    "        if type(element) is list:\n",
    "            for item in element:\n",
    "                flat_list.append(item)\n",
    "        else:\n",
    "            flat_list.append(element)\n",
    "    return flat_list\n",
    "\n",
    "band_channels = list(chain(range(29,35),range(46,60),range(78,96),range(102,106)))\n",
    "\n",
    "window = list(range(56,83))\n",
    "\n",
    "def fit_line(dataframe):\n",
    "    spectra = np.average(dataframe)\n",
    "    my_fit,_ = curve_fit(powerlaw,vims_wave[band_channels],spectra[band_channels],p0=[.15,-12])\n",
    "    final_fit = powerlaw(vims_wave,*my_fit)\n",
    "    return final_fit\n",
    "\n",
    "def custom_fit(dataframe):\n",
    "    spectra = np.average(dataframe)\n",
    "    my_fit,_ = curve_fit(powerlaw,vims_wave[band_channels],spectra[band_channels],p0=[.15,-12])\n",
    "    final_spectra = spectra-powerlaw(vims_wave,*my_fit)\n",
    "    return final_spectra\n",
    "\n",
    "def single_fit(spectrum):\n",
    "    my_fit,_ = curve_fit(powerlaw,vims_wave[band_channels],spectrum[band_channels],p0=[.15,-12])\n",
    "    final_spectrum = spectrum-powerlaw(vims_wave,*my_fit)\n",
    "\n",
    "def shift_ret(spectrum):\n",
    "    my_fit,_ = curve_fit(gaussian,vims_wave[window],spectrum[window],p0=[.05,2.05])\n",
    "    return my_fit[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "List 1: (12655, 256)\n",
      "List 2: (12655, 256)\n",
      "Total FileID List: (12655,)\n"
     ]
    }
   ],
   "source": [
    "spec_list = []\n",
    "for number in new_array:\n",
    "    my_string = 'old_spec'+number+' = df'+number+\"['spectrum']\"\n",
    "    exec(my_string)\n",
    "    new_string = 'spec'+number+' = np.array([np.abs(x) for x in old_spec'+number+'])' # Ensures no negative I/F values\n",
    "    exec(new_string)\n",
    "    spec = 'spec'+number\n",
    "    spec_list.append(locals()[spec])\n",
    "\n",
    "copy_list = []\n",
    "for number in new_array:\n",
    "    my_string = 'copy_spec'+number+' = dc(spec'+number+')'\n",
    "    exec(my_string)\n",
    "    copy_spec = 'copy_spec'+number\n",
    "    copy_list.append(locals()[copy_spec])\n",
    "\n",
    "total_spectrum = []\n",
    "for spec in spec_list:\n",
    "    for spectrum in spec:\n",
    "        total_spectrum.append(np.abs(spectrum))\n",
    "\n",
    "file_list = []\n",
    "for number in new_array:\n",
    "    my_string = 'file'+number+' = df'+number+\"['fileid']\"\n",
    "    exec(my_string)\n",
    "    id = 'file'+number\n",
    "    file_list.append(locals()[id])\n",
    "\n",
    "total_files = []\n",
    "for year in file_list:\n",
    "    for id in year:\n",
    "        total_files.append(id)\n",
    "\n",
    "\n",
    "total_spectrum = np.asarray(total_spectrum)\n",
    "total_spectrum_2 = dc(total_spectrum)\n",
    "total_files = np.asarray(total_files)\n",
    "print('List 1: '+str(np.shape(total_spectrum)))\n",
    "print('List 2: '+str(np.shape(total_spectrum_2))) # Confirms that DeepCopy works correctly\n",
    "\n",
    "print('Total FileID List: '+str(np.shape(total_files)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "424\n"
     ]
    }
   ],
   "source": [
    "print(len(np.unique(total_files)))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.9.13 ('base')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "6c01efea219ea0303b6e6cd1897665114cc62146ed112ea21ec489238fdae255"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}