Merge pull request #11 from hanscgruber/bug/auth-api

Issue #8 prefect api now initializes key; tests pass

src/PrefectInterfaces.jl

@@ -34,6 +34,7 @@ module PrefectInterfaces
 
 abstract type AbstractPrefectInterface end
 abstract type AbstractPrefectBlock <: AbstractPrefectInterface end
+abstract type AbstractPrefectConfig <: AbstractPrefectInterface end
 
 export AbstractPrefectInterface,
  AbstractPrefectBlock,
@@ -50,9 +51,9 @@ export AbstractPrefectInterface,
  getblock,
  makeblock
 
+include("prefectblock/prefectblocktypes.jl")
 include("config.jl")
 include("prefectblock/prefectblock.jl")
-include("prefectblock/prefectblocktypes.jl")
 include("Datasets/Datasets.jl")
 
 using .Datasets

src/config.jl

@@ -1,7 +1,9 @@
 """
- PrefectAPI(url::String) <:AbstractPrefectInterface
+ PrefectAPI(url::String, key::SecretString) <:AbstractPrefectInterface
 
-Mutable struct tha stores the Prefect server api endpoint. All `PrefectInterface` operations depend on connecting to a running Prefect server to pull block information. Constructor with no arguments assigns env variable `PREFECT_API_URL` to url field.
+Mutable struct tha stores the Prefect server api endpoint. All `PrefectInterface` operations depend on connecting to a running Prefect server to pull block information. Constructor with no arguments assigns env variables `PREFECT_API_URL`, `PREFECT_API_KEY`
+ 
+If `PREFECT_API_KEY` does not exist then an empty string is assigned to `key`. For Prefect Server with no authentication (or with auth managed by connection string) the empty key will not interfere with API calls.
 
 # Examples:
 ```jldoctest
@@ -10,33 +12,49 @@ julia> using PrefectInterfaces
 julia> ENV["PREFECT_API_URL"] = "http://127.0.0.1:4300/api";
 
 julia> api = PrefectAPI()
-PrefectAPI("http://127.0.0.1:4300/api")
+PrefectAPI("http://127.0.0.1:4300/api", ####Secret####)
 
 julia> api.url
 "http://127.0.0.1:4300/api"
 
-julia> api.url = "http://127.0.0.1:4333/api"
-"http://127.0.0.1:4333/api"
+julia> api.key.secret
+""
 
-julia> PrefectAPI("http://127.0.0.1:4444/api").url
-"http://127.0.0.1:4444/api"
+julia> api.url = "https://api.prefect.cloud/api/accounts/0eEXAMPLE";
+
+julia> api.key = SecretString("abcd1234")
+####Secret####
+
+julia> api = PrefectAPI("https://api.prefect.cloud/api/accounts/0eEXAMPLE", "abcd1234")
+PrefectAPI("https://api.prefect.cloud/api/accounts/0eEXAMPLE", ####Secret####)
 ```
 """
-mutable struct PrefectAPI <: AbstractPrefectInterface
+mutable struct PrefectAPI <: AbstractPrefectConfig
  url::AbstractString
+ key::SecretString
+ PrefectAPI(url, key) = new(url, SecretString(key))
 end
+PrefectAPI(url) = PrefectAPI(url, "")
 PrefectAPI() = begin
- if haskey(ENV, "PREFECT_API_URL")
+ if ! haskey(ENV, "PREFECT_API_URL")
+ @warn "Prefect API URL is needed to call Prefect Server.\n" *
+ "Zero-argument constructor requires ENV value PREFECT_API_URL optional PREFECT_API_KEY."
+ end
+ if ! haskey(ENV, "PREFECT_API_KEY")
  PrefectAPI(ENV["PREFECT_API_URL"])
  else
- @warn "Prefect API URL is needed to call Prefect Server.\n" *
- "Set ENV variable PREFECT_API_URL or provide endpoint URL to this constructor."
- throw(KeyError("PREFECT_API_URL"))
+ PrefectAPI(ENV["PREFECT_API_URL"], ENV["PREFECT_API_KEY"])
  end
 end
 
 
 # TODO: so far not used anywhere. idea is a type to load env with dotenv when contstructor is called.
-struct PrefectConfig <: AbstractPrefectInterface
+# maybe PrefectConfig is a global struct to hold config and is accessible without passing in and 
+# out of a bunch of function calls. simply the model and changes get easier.
+# it makes it more sensible when handing off from python shell call to an included file
+# you cant pass parameters to an included file but you want access to parameters passed from 
+# prefect flows.
+# Every PrefectInterfaces session has a PrefectConfig.init() action or some initialization.
+struct PrefectConfig <: AbstractPrefectConfig
  env::Dict
 end

src/prefectblock/prefectblock.jl

@@ -7,11 +7,11 @@ using Parameters, Match
 Constructors:
 
  PrefectBlock(blockname::String)
- PrefectBlock(blockname::String, api_url::String)
+ PrefectBlock(blockname::String, api::PrefectAPI)
  PrefectBlock(blockname::String, block::AbstractPrefectBlock)
 
 Returns a Prefect Block from the Prefect server, the block data is stored in the `block` field. Prefect Block names are strings called 'slugs', formatted as `block-type-name/block-name`.
-A Prefect Block is uniquely specified by its name and the Prefect DB where it is stored, therefore the API URL is necessary for the constructor.
+A Prefect Block is uniquely specified by its name and the Prefect DB where it is stored, therefore the API URL is necessary for the constructor. The single-argument constructor takes the default PrefectAPI() as the Prefect Server endpoint.
 
 A non-server block can be constructed by supplying an AbstractPrefectBlock object.
 
@@ -40,24 +40,22 @@ struct PrefectBlock <: AbstractPrefectBlock
  block::AbstractPrefectBlock
 end
 PrefectBlock(blockname::String) = begin
- url = PrefectAPI().url
- blockdict = getblock(blockname, api_url=url)
+ blockdict = getblock(blockname)
  newblock = makeblock(blockdict)
  @assert blockname == newblock.blockname
  PrefectBlock(blockname, newblock)
 end
-PrefectBlock(blockname::String, api_url::String) = begin
- url = PrefectAPI(api_url).url
- block_dict = getblock(blockname, api_url=url)
+PrefectBlock(blockname::String, api::PrefectAPI) = begin
+ block_dict = getblock(blockname, api = api)
  newblock = makeblock(block_dict)
  @assert blockname == newblock.blockname
  PrefectBlock(blockname, newblock)
 end
 
 """
- ls(; type="block", api_url::String=PrefectAPI().url)
+ ls(; type="block", api::PrefectAPI = PrefectAPI())
 
-Calls the Prefect server and returns a list of all defined blocks as Vector{String}. Default is to list all blocks, later implementation could include "flows", "deployments", "work-pool" etc.
+Calls the Prefect server and returns a list of all defined blocks as Vector{String}. Default is to list all blocks, later implementation could include "flows", "deployments", "work-pool" etc. See `PrefectAPI` docs for details about authentication if needed.
 
 # Examples:
 ```julia
@@ -78,24 +76,25 @@ julia> ls()
  "string/environment"
 ```
 """
-function ls(; type="block", api_url::String=PrefectAPI().url)
+function ls(; type="block", api::PrefectAPI = PrefectAPI())
  # TODO: this could be better, have a show definition and return PrefectBlockList, as a dict of julia block type or block id.
  # TODO: deployments, flows, flow-run, etc.
  if type ∉ ["block"]
  @warn """Only type="block" currently supported""" type
  return nothing
  end
  response = try
- HTTP.request(
- "POST"
- , "$(api_url)/block_documents/filter"
+ HTTP.post(
+ "$(api.url)/block_documents/filter"
+ , ["Authorization" => "Bearer $(api.key.secret)"]
+ ; copyheaders = false
  , connect_timeout = 3
  , readtimeout = 5
  , retries = 1
  )
  catch ex
  if typeof(ex) ∈ [HTTP.Exceptions.StatusError, HTTP.ConnectError]
- @warn "no connection." api_url
+ @warn "no connection." api.url
  println("$ex")
  return nothing
  else
@@ -112,17 +111,18 @@ function ls(; type="block", api_url::String=PrefectAPI().url)
 end
 
 """
- getblock(blockname::String; api_url::String=PrefectAPI().url)
+ getblock(blockname::String; api::AbstractPrefectConfig = PrefectAPI())
 
-Makes an `HTTP.get()` call to provided URL endpoint, default endpoint constructed by `PrefectAPI().url`. Returns a Dict containing the Prefect Block specification.
+Makes an `HTTP.get()` call to provided URL endpoint, default endpoint constructed by `PrefectAPI`. Returns a Dict containing the Prefect Block specification.
 """
-function getblock(blockname::String; api_url::String=PrefectAPI().url)
+function getblock(blockname::String; api::PrefectAPI = PrefectAPI())
  block = blockname_components(blockname)
  try
- response = HTTP.request(
- "GET"
- , "$(api_url)/block_types/slug/$(block.slug)/block_documents/name/$(block.name)"
- ; query = ["include_secrets" => "true"]
+ response = HTTP.get(
+ "$(api.url)/block_types/slug/$(block.slug)/block_documents/name/$(block.name)"
+ , ["Authorization" => "Bearer $(api.key.secret)"]
+ ; copyheaders = false
+ , query = ["include_secrets" => "true"]
  , connect_timeout = 3
  , readtimeout = 5
  , retries = 1

src/prefectblock/prefectblocktypes.jl

@@ -32,6 +32,30 @@ end
 Base.show(io::IO, s::SecretString) = print(io, "####Secret####")
 Base.dump(io::IOContext, s::SecretString, n::Int64, indent) = print(io, "SecretString")
 
+"""
+ SecretBlock(blockname::String, blocktype::String, value::SecretString) <: AbstractPrefectBlock
+
+A struct for storing a Prefect Block with a SecretString as value field. This permits retrieving secrets from the Prefect Server/Prefect Cloud. The secret field is accessible via the `value.secret` field.
+
+*NOTE:* This is not an ecrypted secrets store, it is a log obfuscator.
+
+# Example:
+```jldoctest
+julia> using PrefectInterfaces
+
+julia> secretblock = SecretBlock("secret", "necromancer", "abcd1234")
+SecretBlock("secret", "necromancer", ####Secret####)
+
+julia> dump(secretblock, maxdepth = 10)
+SecretBlock
+ blockname: String "secret"
+ blocktype: String "necromancer"
+ value: SecretString
+
+julia> secretblock.value.secret
+"abcd1234"
+```
+"""
 struct SecretBlock <: AbstractPrefectBlock
  blockname::String
  blocktype::String

test/config/config.jl

@@ -1,11 +1,22 @@
 using PrefectInterfaces
 using PrefectInterfaces.Datasets
 
-@test PrefectAPI("http://127.0.0.1:4444/api").url == "http://127.0.0.1:4444/api"
+# PrefectAPI
 api = PrefectAPI()
 @test api.url == "http://127.0.0.1:4300/api"
 @test typeof(api) == PrefectAPI
 
+ # single arg constructor
+@test PrefectAPI("http://127.0.0.1:4444/api").url == "http://127.0.0.1:4444/api"
+@test PrefectAPI("http://127.0.0.1:4444/api").key.secret == ""
+
+ # url and key argument constructor
+api = PrefectAPI("https://api.prefect.cloud/api/accounts/0eEXAMPLE", "abcd1234")
+@test api.url == "https://api.prefect.cloud/api/accounts/0eEXAMPLE"
+@test repr(api.key) == "####Secret####"
+@test api.key.secret == "abcd1234"
+
+# Datasets
 dst = Datasets.PrefectDatastoreNames()
 @test propertynames(dst) == (:remote, :local)
 @test dst.remote == "s3-bucket/willowdata"

todo/DONE.md

@@ -37,3 +37,34 @@ x = settings.PREFECT_API_URL.value()
 'http://127.0.0.1:4300/api'
 ```
 So it works. The julia application needs to receive this value as a parameter.
+
+## DONE: SECRETBLOCK OBFUSCATE DUMP
+Issue #9
+
+DONE: override dump to obscure SecretBlock from logs
+```jl
+sblk = SecretBlock("poo", "patype", "abc123")
+SecretBlock("poo", "patype", ####Secret####)
+
+show(sblk)
+SecretBlock("poo", "patype", ####Secret####)
+dump(sblk)
+SecretBlock
+ blockname: String "poo"
+ blocktype: String "patype"
+ value: SecretString
+ secret: String "abc123"
+
+Base.dump(io::IOContext, s::SecretString, n::Int64, indent) = print(io, "SecretString")
+dump(sblk)
+# SecretBlock
+# blockname: String "poo"
+# blocktype: String "patype"
+# value: SecretString
+dump(sblk, maxdepth = 4)
+# SecretBlock
+# blockname: String "poo"
+# blocktype: String "patype"
+# value: SecretString
+```
+DONE

todo/TODO.md

@@ -0,0 +1,4 @@
+# TODO
+Not quite issues, but on the list.
+There are some sprinkled throughout the repo.
+