From e77d2e01ef8e8cebb89608ccb984946dd9a6b469 Mon Sep 17 00:00:00 2001 From: AmitHasanShuvo Date: Thu, 19 Aug 2021 23:32:04 +0600 Subject: [PATCH] initial commit --- .gitignore | 4 + CHANGELOG.txt | 26 ++++ LICENCE.txt | 7 + LICENSE | 21 --- MANIFEST.in | 1 + README.md | 41 +++++- data_inspector/__init__.py | 1 + data_inspector/data_inspector.py | 228 +++++++++++++++++++++++++++++++ setup.py | 22 +++ 9 files changed, 328 insertions(+), 23 deletions(-) create mode 100644 CHANGELOG.txt create mode 100644 LICENCE.txt delete mode 100644 LICENSE create mode 100644 MANIFEST.in create mode 100644 data_inspector/__init__.py create mode 100644 data_inspector/data_inspector.py create mode 100644 setup.py diff --git a/.gitignore b/.gitignore index 5d947ca..50bc796 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,10 @@ bin-release/ # Other files and folders .settings/ +.build/ +.dist/ +.data_inspector.egg-info/ + # Executables *.swf diff --git a/CHANGELOG.txt b/CHANGELOG.txt new file mode 100644 index 0000000..4ceb1d1 --- /dev/null +++ b/CHANGELOG.txt @@ -0,0 +1,26 @@ +Change Log +========== + +0.0.1 (20/08/2021) +------------------ +- First Release + +0.0.2 (20/08/2021) +------------------ +- Minor updates + +0.0.3 (20/08/2021) +------------------ +- Minor updates + +0.0.4 (20/08/2021) +------------------ +- Minor updates + +0.0.5 (20/08/2021) +------------------ +- Minor updates + +0.0.6 (20/08/2021) +------------------ +- Minor updates \ No newline at end of file diff --git a/LICENCE.txt b/LICENCE.txt new file mode 100644 index 0000000..06e0c4d --- /dev/null +++ b/LICENCE.txt @@ -0,0 +1,7 @@ +Copyright 2021 Kazi Amit Hasan + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/LICENSE b/LICENSE deleted file mode 100644 index 4292372..0000000 --- a/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -MIT License - -Copyright (c) 2021 Kazi Amit Hasan - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..2ee8ab3 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1 @@ +global-include *.txt *.py \ No newline at end of file diff --git a/README.md b/README.md index 0a2c4d2..b2b90cc 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,39 @@ -# data-inspector -This module brings different functions to make EDA, data cleaning easier. +# Data Inspector +## This module brings different functions to make EDA, data cleaning easier. +### Author: Kazi Amit Hasan + +## Project Description: + +Data Inspector brings a total of 15 essential exploratory data analysis, data cleaning automations to make a dataset understandable. This is a perfect tool to get started with you data. + +data inspector helps to make +## Installation + +```pip install data-inspector``` + +Package available at https://pypi.org/project/data-inspector/ + + +### Available automation: + + +1. Line plot : ```line_plot(data, x_data, y_data, x_label="", y_label="", title="")``` +2. Skew feature: ```plot_skewed_feature(data, column)``` +3. Showing data distribution: ```show_distribution(data, column)``` +4. Scatter plot: ```plot_scatter(data,x_data, y_data)``` +5. Correlation plot: ```plot_correlation(data)``` +6. Create histogram: ```histogram(data,column, x_label, y_label, title)``` +7. Create bar plot: ```plot_bar(data, column, xlabel, ylabel, title)``` +8. Create boxplots of all features: ```box_plot(data)``` +9. Checking dataset's shape: ```datasetShape(data)``` +10. Get dataset's diagnostic plots: ```diagnostic_plots(data, variable)``` +11. Divide numerical and categorical features: ```divideFeatures(data)``` +12. Fill NaN values: ```fillNan(data, column, value)``` +13. Get pearson's correlation between two variables: ```get_correlation(column_1, column_2, data)``` +14. Plotting kde plots:``` plot_cont_kde(data, var)``` +15. Automatic calculating the missing values and their percentage along with visualization : ```calculating_missing_values(data)``` + + + + + diff --git a/data_inspector/__init__.py b/data_inspector/__init__.py new file mode 100644 index 0000000..7e839c1 --- /dev/null +++ b/data_inspector/__init__.py @@ -0,0 +1 @@ +from data_inspector.data_inspector import * diff --git a/data_inspector/data_inspector.py b/data_inspector/data_inspector.py new file mode 100644 index 0000000..fa4d4f8 --- /dev/null +++ b/data_inspector/data_inspector.py @@ -0,0 +1,228 @@ +import matplotlib.pyplot as plt +import seaborn as sns +import warnings +import numpy as np +import pandas as pd +from scipy.stats import pearsonr +from scipy import stats + +warnings.filterwarnings("ignore") + +""" +NAME + data_inspector +DESCRIPTION + This module brings different functions to make EDA, data cleaning easier. + +MODULE CONTENTS + 1. line_plot + 2. plot_skewed_feature + 3. show_distribution + 4. plot_scatter + 5. plot_correlation + 6. get_correlation (pearson) + 7. histogram + 8. plot_bar + 9. box_plot + 10. datasetShape + 11. diagnostic_plots + 12. divideFeatures + 13. fillNan + 14. plot_cont_kde + 15. calculating_missing_values +""" + + +# fucntion to plot line_plot +# Reference: https://notebook.community/mzohaib10/datascience-starters/visualization-in-matplotlib/visualization-in-matplotlib + +def line_plot(data, x_data, y_data, x_label="", y_label="", title=""): + # Create the plot object + _, ax = plt.subplots() + + # Plot the best fit line, set the linewidth (lw), color and + # transparency (alpha) of the line + ax.plot(data[x_data], data[y_data], lw = 2, color = '#539caf', alpha = 1) + + # Label the axes and provide a title + ax.set_title(title) + ax.set_xlabel(x_label) + ax.set_ylabel(y_label) + + + +# fucntion to plot sample skewed feature +# Refernce: + +def plot_skewed_feature(data, column): + plt.figure(figsize=(10,4)) + sns.distplot(data[column]) + plt.show() + + +# fucntion to show data distributions + +def show_distribution(data, column): + + fig, ax = plt.subplots(figsize=(10,5)) + sns.histplot(data, x=column, kde=True, ax=ax) + plt.show() + + +# fucntion to show scatter plots +# Reference: https://notebook.community/mzohaib10/datascience-starters/visualization-in-matplotlib/visualization-in-matplotlib + +def plot_scatter(data,x_data, y_data): + + plt.figure(figsize=(15,5)) + plt.scatter(data[x_data], data[y_data], color='r') + plt.title(f'{x_data} vs {y_data}', fontsize=15, fontweight='bold') + plt.xlabel(f'{x_data}', fontsize=15, fontweight='bold') + plt.ylabel(f'{y_data}', fontsize=15, fontweight='bold') + plt.grid() + + +# fucntion to show correlations +# Reference: https://seaborn.pydata.org/examples/many_pairwise_correlations.html + +def plot_correlation(data): + corr = data.corr() + mask = np.zeros_like(corr, dtype=np.bool) + mask[np.triu_indices_from(mask)] = True + sns.heatmap(corr, mask = mask, annot=True) + plt.show() + +# fucntion to get pearson correlations and P value +# Reference: https://github.com/PacktPublishing/Hands-on-Exploratory-Data-Analysis-with-Python/blob/master/Chapter%2011/Chapter11.ipynb + +def get_correlation(column_1, column_2, data): + pearson_corr, p_value = pearsonr(data[column_1], data[column_2]) + print("Correlation between {} and {} is {}".format(column_1, column_2, pearson_corr)) + print("P-value of this correlation is {}".format(p_value)) + + +# fucntion to show histogram +# Reference: https://notebook.community/mzohaib10/datascience-starters/visualization-in-matplotlib/visualization-in-matplotlib + +def histogram(data,column, x_label, y_label, title): + _, ax = plt.subplots() + ax.hist(data[column], color = '#539caf') + ax.set_ylabel(y_label) + ax.set_xlabel(x_label) + ax.set_title(title) + +# fucntion to plot bar +# Reference: https://notebook.community/mzohaib10/datascience-starters/visualization-in-matplotlib/visualization-in-matplotlib + +def plot_bar(data, column, xlabel, ylabel, title): + + ax = data[column].value_counts().sort_values().plot(kind='bar',figsize=(10,5), grid=True) + ax.set( + xlabel = xlabel, + ylabel = ylabel, + title = title) + plt.show() + +# fucntion to create boxplot of all features + + +def box_plot(data): + + plt.subplots(figsize=(15,6)) + data.boxplot(patch_artist=True, sym="k.") + plt.xticks(rotation=90) + +# fucntion to get dataset's columns and rows + +def datasetShape(data): + rows, cols = data.shape + print("The dataframe has",rows,"rows and",cols,"columns.") + +# fucntion to get a total diagnostic result +# Reference: https://www.kaggle.com/asimislam/tutorial-python-subplots + +def diagnostic_plots(data, variable): + # function takes a dataframe (df) and + # the variable of interest as arguments + + # define figure size + plt.figure(figsize=(25, 5)) + + # creating histogram + plt.subplot(1, 5, 1) + sns.histplot(data[variable], bins=30) + plt.title('Histogram') + + # creating Q-Q plot + plt.subplot(1, 5, 2) + stats.probplot(data[variable], dist="norm", plot=plt) + plt.ylabel('Variable quantiles') + + # creating boxplot + plt.subplot(1, 5, 3) + sns.boxplot(y=data[variable]) + plt.title('Boxplot') + + plt.show() + + + +# function to select numerical and categorical features + +def divideFeatures(data): + numerical_features = data.select_dtypes(include=[np.number]) + categorical_features = data.select_dtypes(include=[np.object]) + return numerical_features, categorical_features + + +# fucntion to fill the Nan values with a specific value + +def fillNan(data, column, value): + data[column].fillna(value, inplace = True) + + + + # fucntion to plot kde + # Reference: https://www.analyticsvidhya.com/blog/2020/10/optimizing-exploratory-data-analysis-using-functions-in-python/ + + +def plot_cont_kde(data, var, l=10, b=5): + mini = data[var].min() + maxi = data[var].max() + ran = data[var].max()-data[var].min() + mean = data[var].mean() + skew = data[var].skew() + kurt = data[var].kurtosis() + median = data[var].median() + st_dev = data[var].std() + points = mean-st_dev, mean+st_dev + fig, axes = plt.subplots(1, 2) + sns.boxplot(data=data, x=var, ax=axes[0]) + sns.distplot(a=data[var], ax=axes[1], color='#ff4125') + sns.lineplot(points, [0, 0], color='black', label="std_dev") + sns.scatterplot([mini, maxi], [0, 0], + color='orange', label="min/max") + sns.scatterplot([mean], [0], color='red', label="mean") + sns.scatterplot([median], [0], color='blue', label="median") + fig.set_size_inches(l, b) + plt.title('std_dev = {}; kurtosis = {};nskew = {}; range = {}; nmean = {}; median = {};' + .format((round(points[0],2),round(points[1],2)), + round(kurt,2),round(skew,2),(round(mini,2),round(maxi,2), + round(ran,2)),round(mean,2), round(median,2))) + + +# fucntion to calculate and plot missing values +# Reference: https://stackoverflow.com/questions/51070985/find-out-the-percentage-of-missing-values-in-each-column-in-the-given-dataset + +def calculating_missing_values(data): + missing = data.isna().sum().sort_values(ascending = True) + missing = missing[missing !=0] + missing_percentage = missing/data.shape[0]*100 + # return missing, missing_percentage + if data.isna().any().sum()>0: + missing, missing_percentage = calculating_missing_values(data) + missing.plot(kind ='bar', figsize=(30,10)) + plt.title('Missing Values') + plt.show() + else: + print ('No missing values here') \ No newline at end of file diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..012c457 --- /dev/null +++ b/setup.py @@ -0,0 +1,22 @@ +import setuptools + + +setuptools.setup( + name='data_inspector', + version='0.0.8', + author="Kazi Amit Hasan", + author_email="kaziamithasan89@gmail.com", + description="This module brings different functions to make EDA, data cleaning easier.", + long_description=open('README.md').read() + '\n\n' + open('CHANGELOG.txt').read(), + License='MIT', + long_description_content_type="text/markdown", + url="", + keywords='eda', + packages=setuptools.find_packages(), + classifiers=[ + "Programming Language :: Python :: 3", + "License :: OSI Approved :: MIT License", + "Operating System :: OS Independent", + ], + install_requires=['pandas==1.1.2','matplotlib==3.1.2','numpy==1.18.5', 'seaborn==0.11.1','scipy==1.6.2'] +) \ No newline at end of file