From 892cda6758f7b0918c6117ba6de8cc7ae3ba21bd Mon Sep 17 00:00:00 2001 From: Medha Basu Date: Wed, 10 Jan 2024 17:19:22 +0800 Subject: [PATCH 1/6] made API key visible while it is being typed in the CLI --- defog/cli.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/defog/cli.py b/defog/cli.py index f466737..9ba75a0 100644 --- a/defog/cli.py +++ b/defog/cli.py @@ -94,9 +94,9 @@ def init(): ) api_key = os.environ.get("DEFOG_API_KEY") else: - api_key = getpass.getpass( - prompt="Please enter your DEFOG_API_KEY. You can get it from https://defog.ai/accounts/dashboard/ and creating an account:" - ) + print("Please enter your DEFOG_API_KEY. You can get it from https://defog.ai/accounts/dashboard/ and creating an account:") + api_key = prompt() + # prompt user for db_type print( "What database are you using? Available options are: " From ec1b9247a4a636ad339e3405549801e404f46325 Mon Sep 17 00:00:00 2001 From: Medha Basu Date: Wed, 10 Jan 2024 17:25:27 +0800 Subject: [PATCH 2/6] stripped all prompt input to handle accidental leading or trailing spaces --- defog/cli.py | 48 ++++++++++++++++++++++++------------------------ 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/defog/cli.py b/defog/cli.py index 9ba75a0..03c5a0d 100644 --- a/defog/cli.py +++ b/defog/cli.py @@ -76,7 +76,7 @@ def init(): print( "It looks like you've already initialized defog. Do you want to overwrite your existing configuration? (y/n)" ) - overwrite = prompt() + overwrite = prompt().strip() if overwrite.lower() != "y": print("We'll keep your existing config. No changes were made.") sys.exit(0) @@ -95,32 +95,32 @@ def init(): api_key = os.environ.get("DEFOG_API_KEY") else: print("Please enter your DEFOG_API_KEY. You can get it from https://defog.ai/accounts/dashboard/ and creating an account:") - api_key = prompt() + api_key = prompt().strip() # prompt user for db_type print( "What database are you using? Available options are: " + ", ".join(defog.SUPPORTED_DB_TYPES) ) - db_type = prompt() + db_type = prompt().strip() db_type = db_type.lower() while db_type not in defog.SUPPORTED_DB_TYPES: print( "Sorry, we don't support that database yet. Available options are: " + ", ".join(defog.SUPPORTED_DB_TYPES) ) - db_type = prompt() + db_type = prompt().strip() db_type = db_type.lower() # depending on db_type, prompt user for appropriate db_creds if db_type == "postgres" or db_type == "redshift": print("Please enter your database host:") - host = prompt() + host = prompt().strip() print("Please enter your database port:") - port = prompt() + port = prompt().strip() print("Please enter your database name:") - database = prompt() + database = prompt().strip() print("Please enter your database user:") - user = prompt() + user = prompt().strip() password = getpass.getpass(prompt="Please enter your database password:") db_creds = { "host": host, @@ -132,11 +132,11 @@ def init(): elif db_type == "mysql": print("Please enter your database host:") - host = prompt() + host = prompt().strip() print("Please enter your database name:") - database = prompt() + database = prompt().strip() print("Please enter your database user:") - user = prompt() + user = prompt().strip() print("Please enter your database password:") password = getpass.getpass(prompt="Please enter your database password:") db_creds = { @@ -147,11 +147,11 @@ def init(): } elif db_type == "snowflake": print("Please enter your database account:") - account = prompt() + account = prompt().strip() print("Please enter your database warehouse:") - warehouse = prompt() + warehouse = prompt().strip() print("Please enter your database user:") - user = prompt() + user = prompt().strip() print("Please enter your database password:") password = getpass.getpass(prompt="Please enter your database password:") db_creds = { @@ -162,12 +162,12 @@ def init(): } elif db_type == "databricks": print("Please enter your databricks host:") - host = prompt() + host = prompt().strip() token = getpass.getpass(prompt="Please enter your databricks token:") print("Please add your http_path:") - http_path = prompt() + http_path = prompt().strip() print("Please enter your schema name (this is usually 'default'):") - schema = prompt() + schema = prompt().strip() db_creds = { "server_hostname": host, "access_token": token, @@ -176,7 +176,7 @@ def init(): } elif db_type == "bigquery": print("Please enter your bigquery json key's path:") - json_key_path = prompt() + json_key_path = prompt().strip() db_creds = { "json_key_path": json_key_path, } @@ -194,7 +194,7 @@ def init(): print( "Please enter the names of the tables you would like to register, separated by a space:" ) - table_names = prompt() + table_names = prompt().strip() table_name_list = re.split(r"\s+", table_names.strip()) # if input is empty, exit if table_name_list == [""]: @@ -211,7 +211,7 @@ def init(): print( "You can give us more context about your schema by editing the CSV above. Once you're done, you can just hit enter to upload the data in the spreadsheet to Defog. If you would like to exit instead, just enter `exit`." ) - upload_option = prompt() + upload_option = prompt().strip() if upload_option == "exit": print("Exiting.") sys.exit(0) @@ -265,7 +265,7 @@ def gen(): print( "If you would like to index all of your tables, just leave this blank and hit enter (Supported for postgres + redshift only)." ) - table_names = prompt() + table_names = prompt().strip() table_name_list = re.split(r"\s+", table_names.strip()) else: table_name_list = sys.argv[2:] @@ -292,7 +292,7 @@ def update(): print( "defog update requires a CSV that contains your Database metadata. Please enter the path to the CSV you would like to update:" ) - filename = prompt() + filename = prompt().strip() else: filename = sys.argv[2] # load config from .defog/connection.json @@ -314,7 +314,7 @@ def query(): df = defog.Defog() # load config from .defog/connection.json if len(sys.argv) < 3: print("defog query requires a query. Please enter your query:") - query = prompt() + query = prompt().strip() else: query = sys.argv[2] @@ -432,7 +432,7 @@ def deploy(): # check args for gcp or aws if len(sys.argv) < 3: print("defog deploy requires a cloud provider. Please enter 'gcp' or 'aws':") - cloud_provider = prompt().lower() + cloud_provider = prompt().strip().lower() else: cloud_provider = sys.argv[2].lower() From e12eef27df857d2f0fd3a81ea32d60ceaf66c1fa Mon Sep 17 00:00:00 2001 From: Medha Basu Date: Wed, 10 Jan 2024 17:33:14 +0800 Subject: [PATCH 3/6] replace getpass with pwinput to give users feedback as they type passwords --- defog/cli.py | 10 +++++----- requirements.txt | 3 ++- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/defog/cli.py b/defog/cli.py index 03c5a0d..0f7193e 100644 --- a/defog/cli.py +++ b/defog/cli.py @@ -1,6 +1,6 @@ import datetime import decimal -import getpass +import pwinput import json import os import re @@ -121,7 +121,7 @@ def init(): database = prompt().strip() print("Please enter your database user:") user = prompt().strip() - password = getpass.getpass(prompt="Please enter your database password:") + password = pwinput.pwinput(prompt="Please enter your database password. This won't be displayed as you're typing for privacy reasons") db_creds = { "host": host, "port": port, @@ -138,7 +138,7 @@ def init(): print("Please enter your database user:") user = prompt().strip() print("Please enter your database password:") - password = getpass.getpass(prompt="Please enter your database password:") + password = pwinput.pwinput(prompt="Please enter your database password:") db_creds = { "host": host, "database": database, @@ -153,7 +153,7 @@ def init(): print("Please enter your database user:") user = prompt().strip() print("Please enter your database password:") - password = getpass.getpass(prompt="Please enter your database password:") + password = pwinput.pwinput(prompt="Please enter your database password:") db_creds = { "account": account, "warehouse": warehouse, @@ -163,7 +163,7 @@ def init(): elif db_type == "databricks": print("Please enter your databricks host:") host = prompt().strip() - token = getpass.getpass(prompt="Please enter your databricks token:") + token = pwinput.pwinput(prompt="Please enter your databricks token:") print("Please add your http_path:") http_path = prompt().strip() print("Please enter your schema name (this is usually 'default'):") diff --git a/requirements.txt b/requirements.txt index aaeaae3..eeb1035 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,5 @@ requests>=2.28.2 psycopg2-binary>=2.9.5 pandas -tabulate \ No newline at end of file +tabulate +pwinput \ No newline at end of file From 3c74815375fd8b053d3922768bbac3ab2cf60419 Mon Sep 17 00:00:00 2001 From: Medha Basu Date: Wed, 10 Jan 2024 17:40:31 +0800 Subject: [PATCH 4/6] checks if host has http or https for postgres and redshift and remove it --- defog/cli.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/defog/cli.py b/defog/cli.py index 0f7193e..9d346d2 100644 --- a/defog/cli.py +++ b/defog/cli.py @@ -115,6 +115,9 @@ def init(): if db_type == "postgres" or db_type == "redshift": print("Please enter your database host:") host = prompt().strip() + # check if host has http:// or https:// and remove it if it does + if host.startswith("http://") or host.startswith("https://"): + host = host.split("://")[1] print("Please enter your database port:") port = prompt().strip() print("Please enter your database name:") From 6d99ffa4e8e11ce083f4312695858959b4c0377f Mon Sep 17 00:00:00 2001 From: Medha Basu Date: Wed, 10 Jan 2024 17:49:22 +0800 Subject: [PATCH 5/6] add link to metadata cookbook and give user feedback when schema upload has begun --- defog/cli.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/defog/cli.py b/defog/cli.py index 9d346d2..5eeb615 100644 --- a/defog/cli.py +++ b/defog/cli.py @@ -212,13 +212,14 @@ def init(): print(f"\033[1m{filename}\033[0m\n") print( - "You can give us more context about your schema by editing the CSV above. Once you're done, you can just hit enter to upload the data in the spreadsheet to Defog. If you would like to exit instead, just enter `exit`." + "You can give us more context about your schema by editing the CSV above. Refer to our cookbook on how to do this here: https://defog.notion.site/Cookbook-for-schema-definitions-1650a6855ea447fdb0be75d39975571b#2ba1d37e243e4da3b8f17590b4a3e4e3.\n\nOnce you're done, you can just hit enter to upload the data in the spreadsheet to Defog. If you would like to exit instead, just enter `exit`." ) upload_option = prompt().strip() if upload_option == "exit": print("Exiting.") sys.exit(0) else: + print("We are now uploading this schema to Defog. This might take up to 30 seconds...") resp = df.update_db_schema_csv(filename) if resp["status"] == "success": print("Your schema has been updated. You're ready to start querying!") From 56cf7c7b09bbcc808aa51e8513028f35e4d05a13 Mon Sep 17 00:00:00 2001 From: Medha Basu Date: Wed, 10 Jan 2024 17:52:35 +0800 Subject: [PATCH 6/6] formatting --- defog/cli.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/defog/cli.py b/defog/cli.py index 5eeb615..60b14ef 100644 --- a/defog/cli.py +++ b/defog/cli.py @@ -94,9 +94,11 @@ def init(): ) api_key = os.environ.get("DEFOG_API_KEY") else: - print("Please enter your DEFOG_API_KEY. You can get it from https://defog.ai/accounts/dashboard/ and creating an account:") + print( + "Please enter your DEFOG_API_KEY. You can get it from https://defog.ai/accounts/dashboard/ and creating an account:" + ) api_key = prompt().strip() - + # prompt user for db_type print( "What database are you using? Available options are: " @@ -124,7 +126,9 @@ def init(): database = prompt().strip() print("Please enter your database user:") user = prompt().strip() - password = pwinput.pwinput(prompt="Please enter your database password. This won't be displayed as you're typing for privacy reasons") + password = pwinput.pwinput( + prompt="Please enter your database password. This won't be displayed as you're typing for privacy reasons" + ) db_creds = { "host": host, "port": port, @@ -219,7 +223,9 @@ def init(): print("Exiting.") sys.exit(0) else: - print("We are now uploading this schema to Defog. This might take up to 30 seconds...") + print( + "We are now uploading this schema to Defog. This might take up to 30 seconds..." + ) resp = df.update_db_schema_csv(filename) if resp["status"] == "success": print("Your schema has been updated. You're ready to start querying!")