From 084c04142da052d2abca4be37cba74db9eb8e116 Mon Sep 17 00:00:00 2001 From: Cleop Date: Mon, 26 Feb 2018 17:41:28 +0000 Subject: [PATCH 01/23] #31 Broken code with notes in comments --- config/config.exs | 10 ++++++ config/dev.exs | 2 ++ config/prod.exs | 2 ++ lib/app_web/controllers/aws/s3.ex | 17 +++++++++ lib/app_web/controllers/event_controller.ex | 6 ++++ lib/app_web/controllers/upload_controller.ex | 37 ++++++++++++++++++++ lib/app_web/router.ex | 9 +++++ mix.exs | 8 ++++- mix.lock | 4 +++ 9 files changed, 94 insertions(+), 1 deletion(-) create mode 100644 lib/app_web/controllers/aws/s3.ex create mode 100644 lib/app_web/controllers/upload_controller.ex diff --git a/config/config.exs b/config/config.exs index 35c90b4..1c8a81d 100644 --- a/config/config.exs +++ b/config/config.exs @@ -22,6 +22,16 @@ config :logger, :console, format: "$time $metadata[$level] $message\n", metadata: [:request_id] +# Configure :ex_aws - Taken from JC tutorial +config :ex_aws, + access_key_id: System.get_env("AWS_ACCESS_KEY_ID"), + secret_access_key: System.get_env("AWS_SECRET_ACCESS_KEY"), + s3: [ + scheme: "https://", + host: "dwyl-github-backup.s3.amazonaws.com", + region: "eu-west-2" + ] + # Import environment specific config. This must remain at the bottom # of this file so it overrides the configuration defined above. import_config "#{Mix.env}.exs" diff --git a/config/dev.exs b/config/dev.exs index c986be7..084542e 100644 --- a/config/dev.exs +++ b/config/dev.exs @@ -58,3 +58,5 @@ config :app, App.Repo, pool_size: 10 config :app, :github_api, AppWeb.GithubAPI.HTTPClient +# Is this needed? Thought it might be as HTTPClient is? +config :app, :aws, AppWeb.AWS.S3 diff --git a/config/prod.exs b/config/prod.exs index 9ee2f30..064b01a 100644 --- a/config/prod.exs +++ b/config/prod.exs @@ -26,6 +26,8 @@ config :app, App.Repo, ssl: true config :app, :github_api, AppWeb.GithubAPI.HTTPClient +# Is this needed? Thought it might be as HTTPClient is... +config :app, :aws, AppWeb.AWS.S3 # Do not print debug messages in production config :logger, level: :info diff --git a/lib/app_web/controllers/aws/s3.ex b/lib/app_web/controllers/aws/s3.ex new file mode 100644 index 0000000..3ce0f1e --- /dev/null +++ b/lib/app_web/controllers/aws/s3.ex @@ -0,0 +1,17 @@ +defmodule AppWeb.AWS.S3 do + @moduledoc """ + wrapper functions for saving data to aws s3 + """ + alias ExAws.S3 + + + def saveToS3(payload) do + # Take the payload and then strip the content out of it + IO.inspect payload["issue"]["body"] + # Save the body into s3 + s3Bucket = System.get_env("S3_BUCKET_NAME") + # Code taken from the ExAws repo apart from the inspect + S3.list_objects(s3Bucket) |> ExAws.request |> IO.inspect + end + +end diff --git a/lib/app_web/controllers/event_controller.ex b/lib/app_web/controllers/event_controller.ex index 0d307fd..23a0819 100644 --- a/lib/app_web/controllers/event_controller.ex +++ b/lib/app_web/controllers/event_controller.ex @@ -1,6 +1,7 @@ defmodule AppWeb.EventController do use AppWeb, :controller alias AppWeb.EventType + alias AppWeb.AWS.S3 @github_api Application.get_env(:app, :github_api) @@ -8,6 +9,11 @@ defmodule AppWeb.EventController do headers = Enum.into(conn.req_headers, %{}) x_github_event = Map.get(headers, "x-github-event") +# Was originally tryig to get the payload data in order to save it in here. Since +# Doing JC's tutorial I now also have an upload controller and so perhaps this +# should happen there? + S3.saveToS3(payload) + case EventType.get_event_type(x_github_event, payload["action"]) do :new_installation -> token = @github_api.get_installation_token(payload["installation"]["id"]) diff --git a/lib/app_web/controllers/upload_controller.ex b/lib/app_web/controllers/upload_controller.ex new file mode 100644 index 0000000..88784c0 --- /dev/null +++ b/lib/app_web/controllers/upload_controller.ex @@ -0,0 +1,37 @@ +# Code from JC tutorial + +defmodule AppWeb.UploadController do + use AppWeb.Web, :controller + alias AppWeb.Upload + + def new(conn, _params) do + changeset = Upload.changeset(%Upload{}) + render conn, "new.html", changeset: changeset + end + + def create(conn, %{"upload" => %{"image" => image_params} = upload_params}) do + file_uuid = UUID.uuid4(:hex) + image_filename = image_params.filename + unique_filename = "#{file_uuid}-#{image_filename}" + {:ok, image_binary} = File.read(image_params.path) + bucket_name = System.get_env("BUCKET_NAME") + image = + ExAws.S3.put_object(bucket_name, unique_filename, image_binary) + |> ExAws.request! + + # build the image url and add to the params to be stored + updated_params = + upload_params + |> Map.update(image, image_params, fn _value -> "https://#{bucket_name}.s3.amazonaws.com/#{bucket_name}/#{unique_filename}" end) + changeset = Upload.changeset(%Upload{}, updated_params) + + case Repo.insert!(changeset) do + {:ok, upload} -> + conn + |> put_flash(:info, "Image uploaded successfully!") + |> redirect(to: upload_path(conn, :new)) + {:error, changeset} -> + render conn, "new.html", changeset: changeset + end + end +end diff --git a/lib/app_web/router.ex b/lib/app_web/router.ex index 85bf568..928cf0c 100644 --- a/lib/app_web/router.ex +++ b/lib/app_web/router.ex @@ -24,6 +24,15 @@ defmodule AppWeb.Router do post "/new", EventController, :new end + +# Code is the wrong color and is erroring, not sure why. Tried putting this +# inside the existing scope for '/' but it still didn't work +# Code from JC tutorial + scope “/”, AppWeb do + pipe_through :browser + resources “/upload”, UploadController, only: [:create, :new] + end + # Other scopes may use custom stacks. # scope "/api", AppWeb do # pipe_through :api diff --git a/mix.exs b/mix.exs index 0bad98d..9801076 100644 --- a/mix.exs +++ b/mix.exs @@ -49,7 +49,13 @@ defmodule App.Mixfile do {:credo, "~> 0.3", only: [:dev, :test]}, {:pre_commit, "~> 0.2.4", only: :dev}, {:joken, "~> 1.5"}, - {:jose, "~> 1.8"} + {:jose, "~> 1.8"}, + {:ex_aws, "~> 2.0"}, + {:ex_aws_s3, "~> 2.0"}, + # Not sure whether needed or not + # {:hackney, "~> 1.9"}, + {:sweet_xml, "~> 0.6"}, + {:uuid, "~> 1.1" } ] end diff --git a/mix.lock b/mix.lock index be17358..d7113b8 100644 --- a/mix.lock +++ b/mix.lock @@ -9,6 +9,8 @@ "db_connection": {:hex, :db_connection, "1.1.3", "89b30ca1ef0a3b469b1c779579590688561d586694a3ce8792985d4d7e575a61", [:mix], [{:connection, "~> 1.0.2", [hex: :connection, optional: false]}, {:poolboy, "~> 1.5", [hex: :poolboy, optional: true]}, {:sbroker, "~> 1.0", [hex: :sbroker, optional: true]}]}, "decimal": {:hex, :decimal, "1.4.1", "ad9e501edf7322f122f7fc151cce7c2a0c9ada96f2b0155b8a09a795c2029770", [:mix], []}, "ecto": {:hex, :ecto, "2.2.8", "a4463c0928b970f2cee722cd29aaac154e866a15882c5737e0038bbfcf03ec2c", [:mix], [{:db_connection, "~> 1.1", [hex: :db_connection, optional: true]}, {:decimal, "~> 1.2", [hex: :decimal, optional: false]}, {:mariaex, "~> 0.8.0", [hex: :mariaex, optional: true]}, {:poison, "~> 2.2 or ~> 3.0", [hex: :poison, optional: true]}, {:poolboy, "~> 1.5", [hex: :poolboy, optional: false]}, {:postgrex, "~> 0.13.0", [hex: :postgrex, optional: true]}, {:sbroker, "~> 1.0", [hex: :sbroker, optional: true]}]}, + "ex_aws": {:hex, :ex_aws, "2.0.2", "8df2f96f58624a399abe5a0ce26db648ee848aca6393b9c65c939ece9ac07bfa", [:mix], [{:configparser_ex, "~> 2.0", [hex: :configparser_ex, optional: true]}, {:hackney, "1.6.3 or 1.6.5 or 1.7.1 or 1.8.6 or ~> 1.9", [hex: :hackney, optional: true]}, {:jsx, "~> 2.8", [hex: :jsx, optional: true]}, {:poison, ">= 1.2.0", [hex: :poison, optional: true]}, {:sweet_xml, "~> 0.6", [hex: :sweet_xml, optional: true]}, {:xml_builder, "~> 0.1.0", [hex: :xml_builder, optional: true]}]}, + "ex_aws_s3": {:hex, :ex_aws_s3, "2.0.0", "404e7951820d79774cb67b76a1576ad955dcc44773be5a75fbb0b5bc4278a524", [:mix], [{:ex_aws, "~> 2.0.0", [hex: :ex_aws, optional: false]}]}, "excoveralls": {:hex, :excoveralls, "0.8.1", "0bbf67f22c7dbf7503981d21a5eef5db8bbc3cb86e70d3798e8c802c74fa5e27", [:mix], [{:exjsx, ">= 3.0.0", [hex: :exjsx, optional: false]}, {:hackney, ">= 0.12.0", [hex: :hackney, optional: false]}]}, "exjsx": {:hex, :exjsx, "4.0.0", "60548841e0212df401e38e63c0078ec57b33e7ea49b032c796ccad8cde794b5c", [:mix], [{:jsx, "~> 2.8.0", [hex: :jsx, optional: false]}]}, "file_system": {:hex, :file_system, "0.2.4", "f0bdda195c0e46e987333e986452ec523aed21d784189144f647c43eaf307064", [:mix], []}, @@ -34,5 +36,7 @@ "pre_commit": {:hex, :pre_commit, "0.2.4", "e26feaa149d55d3a6f14ca1340bd8738942d98405de389c0b3c7d48e71e62d66", [:mix], []}, "ranch": {:hex, :ranch, "1.3.2", "e4965a144dc9fbe70e5c077c65e73c57165416a901bd02ea899cfd95aa890986", [:rebar3], []}, "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.1", "28a4d65b7f59893bc2c7de786dec1e1555bd742d336043fe644ae956c3497fbe", [:make, :rebar], []}, + "sweet_xml": {:hex, :sweet_xml, "0.6.5", "dd9cde443212b505d1b5f9758feb2000e66a14d3c449f04c572f3048c66e6697", [:mix], []}, "unicode_util_compat": {:hex, :unicode_util_compat, "0.3.1", "a1f612a7b512638634a603c8f401892afbf99b8ce93a45041f8aaca99cadb85e", [:rebar3], []}, + "uuid": {:hex, :uuid, "1.1.8", "e22fc04499de0de3ed1116b770c7737779f226ceefa0badb3592e64d5cfb4eb9", [:mix], []}, } From 5dc3126b867b88cff6c2fdbd2d5a4bdbe1ca4020 Mon Sep 17 00:00:00 2001 From: Cleop Date: Tue, 27 Feb 2018 14:52:55 +0000 Subject: [PATCH 02/23] #31 Comments out JC tutorial, adds config --- config/config.exs | 6 ++ lib/app_web/controllers/aws/s3.ex | 7 +- lib/app_web/controllers/event_controller.ex | 2 +- lib/app_web/controllers/page_controller.ex | 3 + lib/app_web/controllers/upload_controller.ex | 74 ++++++++++---------- lib/app_web/router.ex | 8 +-- 6 files changed, 56 insertions(+), 44 deletions(-) diff --git a/config/config.exs b/config/config.exs index 1c8a81d..d5af5d7 100644 --- a/config/config.exs +++ b/config/config.exs @@ -22,7 +22,13 @@ config :logger, :console, format: "$time $metadata[$level] $message\n", metadata: [:request_id] +# config :ex_aws, +# access_key_id: [{:system, System.get_env("AWS_ACCESS_KEY_ID")}, :instance_role], +# secret_access_key: [{:system, System.get_env("AWS_SECRET_ACCESS_KEY")}, :instance_role], +# region: "eu-west-2" + # Configure :ex_aws - Taken from JC tutorial +# Or take the version from ex-aws? with instance_role config :ex_aws, access_key_id: System.get_env("AWS_ACCESS_KEY_ID"), secret_access_key: System.get_env("AWS_SECRET_ACCESS_KEY"), diff --git a/lib/app_web/controllers/aws/s3.ex b/lib/app_web/controllers/aws/s3.ex index 3ce0f1e..dbf967e 100644 --- a/lib/app_web/controllers/aws/s3.ex +++ b/lib/app_web/controllers/aws/s3.ex @@ -8,9 +8,12 @@ defmodule AppWeb.AWS.S3 do def saveToS3(payload) do # Take the payload and then strip the content out of it IO.inspect payload["issue"]["body"] - # Save the body into s3 + end + + def get_files_bucket() do s3Bucket = System.get_env("S3_BUCKET_NAME") - # Code taken from the ExAws repo apart from the inspect + # Code taken from the ExAws repo apart from the inspect to list objects in + # the bucket S3.list_objects(s3Bucket) |> ExAws.request |> IO.inspect end diff --git a/lib/app_web/controllers/event_controller.ex b/lib/app_web/controllers/event_controller.ex index 23a0819..44b8ef4 100644 --- a/lib/app_web/controllers/event_controller.ex +++ b/lib/app_web/controllers/event_controller.ex @@ -12,7 +12,7 @@ defmodule AppWeb.EventController do # Was originally tryig to get the payload data in order to save it in here. Since # Doing JC's tutorial I now also have an upload controller and so perhaps this # should happen there? - S3.saveToS3(payload) + # S3.saveToS3(payload) case EventType.get_event_type(x_github_event, payload["action"]) do :new_installation -> diff --git a/lib/app_web/controllers/page_controller.ex b/lib/app_web/controllers/page_controller.ex index c71bf08..3ad7bc7 100644 --- a/lib/app_web/controllers/page_controller.ex +++ b/lib/app_web/controllers/page_controller.ex @@ -1,7 +1,10 @@ defmodule AppWeb.PageController do use AppWeb, :controller + alias AppWeb.AWS.S3 + def index(conn, _params) do + S3.get_files_bucket() render conn, "index.html" end end diff --git a/lib/app_web/controllers/upload_controller.ex b/lib/app_web/controllers/upload_controller.ex index 88784c0..d624097 100644 --- a/lib/app_web/controllers/upload_controller.ex +++ b/lib/app_web/controllers/upload_controller.ex @@ -1,37 +1,37 @@ -# Code from JC tutorial - -defmodule AppWeb.UploadController do - use AppWeb.Web, :controller - alias AppWeb.Upload - - def new(conn, _params) do - changeset = Upload.changeset(%Upload{}) - render conn, "new.html", changeset: changeset - end - - def create(conn, %{"upload" => %{"image" => image_params} = upload_params}) do - file_uuid = UUID.uuid4(:hex) - image_filename = image_params.filename - unique_filename = "#{file_uuid}-#{image_filename}" - {:ok, image_binary} = File.read(image_params.path) - bucket_name = System.get_env("BUCKET_NAME") - image = - ExAws.S3.put_object(bucket_name, unique_filename, image_binary) - |> ExAws.request! - - # build the image url and add to the params to be stored - updated_params = - upload_params - |> Map.update(image, image_params, fn _value -> "https://#{bucket_name}.s3.amazonaws.com/#{bucket_name}/#{unique_filename}" end) - changeset = Upload.changeset(%Upload{}, updated_params) - - case Repo.insert!(changeset) do - {:ok, upload} -> - conn - |> put_flash(:info, "Image uploaded successfully!") - |> redirect(to: upload_path(conn, :new)) - {:error, changeset} -> - render conn, "new.html", changeset: changeset - end - end -end +# # Code from JC tutorial +# +# defmodule AppWeb.UploadController do +# use AppWeb.Web, :controller +# alias AppWeb.Upload +# +# def new(conn, _params) do +# changeset = Upload.changeset(%Upload{}) +# render conn, "new.html", changeset: changeset +# end +# +# def create(conn, %{"upload" => %{"image" => image_params} = upload_params}) do +# file_uuid = UUID.uuid4(:hex) +# image_filename = image_params.filename +# unique_filename = "#{file_uuid}-#{image_filename}" +# {:ok, image_binary} = File.read(image_params.path) +# bucket_name = System.get_env("BUCKET_NAME") +# image = +# ExAws.S3.put_object(bucket_name, unique_filename, image_binary) +# |> ExAws.request! +# +# # build the image url and add to the params to be stored +# updated_params = +# upload_params +# |> Map.update(image, image_params, fn _value -> "https://#{bucket_name}.s3.amazonaws.com/#{bucket_name}/#{unique_filename}" end) +# changeset = Upload.changeset(%Upload{}, updated_params) +# +# case Repo.insert!(changeset) do +# {:ok, upload} -> +# conn +# |> put_flash(:info, "Image uploaded successfully!") +# |> redirect(to: upload_path(conn, :new)) +# {:error, changeset} -> +# render conn, "new.html", changeset: changeset +# end +# end +# end diff --git a/lib/app_web/router.ex b/lib/app_web/router.ex index 928cf0c..3ae74a5 100644 --- a/lib/app_web/router.ex +++ b/lib/app_web/router.ex @@ -28,10 +28,10 @@ defmodule AppWeb.Router do # Code is the wrong color and is erroring, not sure why. Tried putting this # inside the existing scope for '/' but it still didn't work # Code from JC tutorial - scope “/”, AppWeb do - pipe_through :browser - resources “/upload”, UploadController, only: [:create, :new] - end + # scope “/”, AppWeb do + # pipe_through :browser + # resources “/upload”, UploadController, only: [:create, :new] + # end # Other scopes may use custom stacks. # scope "/api", AppWeb do From c080be246acee09c49f232ad1cfae679f22320e5 Mon Sep 17 00:00:00 2001 From: nelsonic Date: Wed, 7 Mar 2018 13:27:49 +0000 Subject: [PATCH 03/23] update "What?" section of readme to reflect change in focus *away* from a simple script --- README.md | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 2bcf93e..9d27681 100644 --- a/README.md +++ b/README.md @@ -3,24 +3,20 @@ ## Why? -As a person who uses GitHub as their "_single-source-of-truth_" +As a person who uses GitHub as their +["_single-source-of-truth_"](https://en.wikipedia.org/wiki/Single_source_of_truth) I need a backup of GitHub issues so that I can work "offline". (_either when I have no network or GH is "down"_) ## What? -Backup GitHub issues. +GitHub (Issue) Backup is an App that lets you (_unsurprisingly_): -+ a script that lets you login to GitHub using OAuth -+ Use GitHub REST API to retrieve all issues -+ Store issues as "flat files" in subdirectories corresponding to the GitHub URL -e.g: if the issue is `dwyl/github-reference/issues/15` -store it as `dwyl/github-reference/issues/15.json` -+ It does not need to "scale" it just needs to work. read: http://paulgraham.com/ds.html - -Note: initially we do not need a Web-UI to let people backup their issues, -but later we could build UI to make it more useable. +1. Store a backup of the content of a GitHub issue +2. Track the changes/edits made to issue description and comments +3. View all your issues when you or GitHub are +["offline"](https://github.com/dwyl/github-reference/issues/15) ## How? From 139da866f8f6cb82bc167d9ed89ac73a39b00e0d Mon Sep 17 00:00:00 2001 From: nelsonic Date: Wed, 7 Mar 2018 13:33:04 +0000 Subject: [PATCH 04/23] adds badges to readme see: https://github.com/dwyl/repo-badges --- README.md | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/README.md b/README.md index 9d27681..07d34b5 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,19 @@ # github-backup + :back: :arrow_up: A script to backup your GitHub Account/Organisation so you can still work when (they/you are) offline. + +
+ +[![Build Status](https://img.shields.io/travis/dwyl/github-backup/master.svg?style=flat-square)](https://travis-ci.org/dwyl/github-backup) +[![Inline docs](http://inch-ci.org/github/dwyl/github-backup.svg?style=flat-square)](http://inch-ci.org/github/dwyl/github-backup) +[![codecov.io](https://img.shields.io/codecov/c/github/dwyl/github-backup/master.svg?style=flat-square)](http://codecov.io/github/dwyl/github-backup?branch=master) +[![Deps Status](https://beta.hexfaktor.org/badge/all/github/dwyl/github-backup.svg?style=flat-square)](https://beta.hexfaktor.org/github/dwyl/github-backup) +[![HitCount](http://hits.dwyl.io/dwyl/github-backup.svg)](https://github.com/dwyl/github-backup) + +
+ + ## Why? As a person who uses GitHub as their @@ -20,6 +33,10 @@ GitHub (Issue) Backup is an App that lets you (_unsurprisingly_): ## How? +The app is built using the Phoenix Web Framework, +if you are `new` to Phoenix (_or Elixir_), +we have a "tutorials": + + Elixir + Phoenix + Tachyons From 3d735a8f3d733b9d234ca9c66d87ca472c6c52b3 Mon Sep 17 00:00:00 2001 From: nelsonic Date: Wed, 7 Mar 2018 23:13:14 +0000 Subject: [PATCH 05/23] emphasise links to tutorials for stack --- README.md | 104 +++++++++++++++++++++++++++++++++++++++++------------- 1 file changed, 80 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index 07d34b5..b0d0452 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # github-backup -:back: :arrow_up: A script to backup your GitHub Account/Organisation so you can still work when (they/you are) offline. +An App that helps you Backup your GitHub Issues so you can still work when (they/you are) offline.
@@ -16,10 +16,10 @@ ## Why? -As a person who uses GitHub as their -["_single-source-of-truth_"](https://en.wikipedia.org/wiki/Single_source_of_truth) -I need a backup of GitHub issues -so that I can work "offline". +`As a person` who uses **GitHub** as their +["***single-source-of-truth***"](https://en.wikipedia.org/wiki/Single_source_of_truth)
+I `need` a backup of GitHub issues
+`so that` I can work "offline".
(_either when I have no network or GH is "down"_) ## What? @@ -33,33 +33,78 @@ GitHub (Issue) Backup is an App that lets you (_unsurprisingly_): ## How? -The app is built using the Phoenix Web Framework, -if you are `new` to Phoenix (_or Elixir_), -we have a "tutorials": +The app is built using the Phoenix Web Framework +(_using the Elixir programming language_) +and Tachyons UI ("Functional CSS") system.
+While you can _run_ the App without knowing Elixir or Phoenix +(_see instructions below_), +if you want to _understand_ how it works, +we _recommend_ learning. +
+If you are `new` to Phoenix, Elixir, +or to Tachyons (_the UI_) +we have a "_beginner tutorials_" +which will bring you up-to-speed: -+ Elixir -+ Phoenix -+ Tachyons ++ Elixir: +[github.com/dwyl/**learn-elixir**](https://github.com/dwyl/learn-elixir) ++ Phoenix: +[github.com/dwyl/**learn-phoenix-framework**](https://github.com/dwyl/learn-phoenix-framework) ++ Tachyons: +[github.com/dwyl/**learn-tachyons**](https://github.com/dwyl/learn-tachyons) -## Set up -The project is hosted on heroku at: https://github-backup.herokuapp.com/ + -To run it locally, you will need to: -- Create a new Github application. -- Run a phoenix server on your machine. +### Set Up Checklist on `localhost` -You'll need to have installed [Elixir](https://elixir-lang.org/install.html), [Phoenix](https://hexdocs.pm/phoenix/installation.html), and [ngrok](https://ngrok.com/download) if you haven't already. +#### Install Dependencies -> _**Note**: **only** `try` to run this on your computer once -you've understood Elixir & Phoenix._ +To run the App on your `localhost` you will need to have **4 dependencies** _installed_: -#### Create a GitHub application ++ Elixir: +https://github.com/dwyl/learn-elixir#installation ++ PostgreSQL: +https://github.com/dwyl/learn-postgresql#installation ++ Phoenix: +https://hexdocs.pm/phoenix/installation.html ++ ngrok: https://ngrok.com/download +(_so you can share your `localhost` url with GitHub_) -The role of the Github application is to send notifications -when events occur on your repositories. -For example you can get a notification when new issues or pull requests are open. +Once all the necessary dependencies are installed, +we can move on. +(_if you get stuck, please let us know_!) + +#### Clone the App + +Run the following the command to `clone` the app: +```sh +git clone git@github.com:dwyl/github-backup.git && cd github-backup +``` + +#### Required Environment Variables + + + + + +#### Install Elixir (_Application-specific_) Dependencies + +```sh +mix deps.get +``` + + +#### Create a GitHub Application + +The role of the Github application is to +send **notifications** when **events** occur in your repositories.
+For example you can get be notified +when **new issues** or **pull requests** are ***opened***. - Access the [new application settings page](https://github.com/settings/apps/new) on your Github account: Settings -> Developer settings -> Github Apps -> New Github App @@ -133,7 +178,10 @@ The `github-backup` server will receive events from Github and then save the det export GITHUB_APP_ID=YOUR_GITHUB_APP_ID ``` - You can generate a new secret key base with ```mix phoenix.gen.secret```. + You can generate a new secret key base with: + ```sh + mix phx.gen.secret + ``` Then execute the command ```source .env``` which will create your environment variables @@ -179,3 +227,11 @@ Open http://localhost:4000 in your web browser. To test your github-backup server try installing your app onto one of your repos. You should see the payload of the request in the tab you have open in your terminal for the phoenix server: ![image](https://user-images.githubusercontent.com/16775804/36433464-77912686-1654-11e8-8a54-0779992d9e18.png) + + +### Deployment + +The project is hosted on Heroku at: https://github-backup.herokuapp.com + +If you want to know _how_ see:
+https://github.com/dwyl/learn-heroku/blob/master/elixir-phoenix-app-deployment.md From 9a6b684a283b9a6a15ee527d1a2822ca076056e2 Mon Sep 17 00:00:00 2001 From: nelsonic Date: Sat, 10 Mar 2018 08:40:53 +0000 Subject: [PATCH 06/23] adds .env_sample file with required environmnet variables + instructions for #35 --- .env_sample | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 .env_sample diff --git a/.env_sample b/.env_sample new file mode 100644 index 0000000..9eb57b2 --- /dev/null +++ b/.env_sample @@ -0,0 +1,8 @@ +# SECRET_KEY_BASE is required for Auth Cookie: +export SECRET_KEY_BASE=run:mix_phx.gen.secret-in-your-terminal-and-paste-result-here + +# PRIVATE_KEY should be generated in your GitHub App settings +export PRIVATE_KEY=YOUR_PRIVATE_KEY + +# GITHUB_APP_ID is found in the settings of your GitHub app under General > About +export GITHUB_APP_ID=YOUR_GITHUB_APP_ID From 286297582c45d169c4febcd06e12a45e29e5ae2c Mon Sep 17 00:00:00 2001 From: nelsonic Date: Sun, 11 Mar 2018 19:07:02 +0000 Subject: [PATCH 07/23] adds link to https://github.com/dwyl/learn-ngrok in readme. (fills gap in docs!) for #35 --- README.md | 264 ++++++++++++++++++++++++++++++++++-------------------- 1 file changed, 167 insertions(+), 97 deletions(-) diff --git a/README.md b/README.md index b0d0452..1f639c4 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,7 @@ # github-backup -An App that helps you Backup your GitHub Issues so you can still work when (they/you are) offline. +An App that helps you **Backup** your **GitHub Issues** +`so that` you can still **work** when (you/they are) ***offline***.
@@ -9,37 +10,39 @@ An App that helps you Backup your GitHub Issues so you can still work when (they [![Inline docs](http://inch-ci.org/github/dwyl/github-backup.svg?style=flat-square)](http://inch-ci.org/github/dwyl/github-backup) [![codecov.io](https://img.shields.io/codecov/c/github/dwyl/github-backup/master.svg?style=flat-square)](http://codecov.io/github/dwyl/github-backup?branch=master) [![Deps Status](https://beta.hexfaktor.org/badge/all/github/dwyl/github-backup.svg?style=flat-square)](https://beta.hexfaktor.org/github/dwyl/github-backup) +
## Why? -`As a person` who uses **GitHub** as their +`As a person` (_`team of people`_) that uses **GitHub** as their ["***single-source-of-truth***"](https://en.wikipedia.org/wiki/Single_source_of_truth)
I `need` a backup of GitHub issues
-`so that` I can work "offline".
-(_either when I have no network or GH is "down"_) +`so that` I can see **issue history** and/or **work** "**offline**".
+(_either when I have no network or GH is "temporarily offline"_) ## What? GitHub (Issue) Backup is an App that lets you (_unsurprisingly_): -1. Store a backup of the content of a GitHub issue -2. Track the changes/edits made to issue description and comments +1. **Store a backup **of the content of the **GitHub issue(s)** +for a given project/repo. +2. Track the changes/edits made to issue description and comments. 3. View all your issues when you or GitHub are ["offline"](https://github.com/dwyl/github-reference/issues/15) ## How? -The app is built using the Phoenix Web Framework -(_using the Elixir programming language_) +The app is built using the **Phoenix Web Framework**
+(_using the Elixir programming language_)
and Tachyons UI ("Functional CSS") system.
-While you can _run_ the App without knowing Elixir or Phoenix +While you can _run_ the App without knowing Elixir or Phoenix
(_see instructions below_), -if you want to _understand_ how it works, -we _recommend_ learning. +if you want to _understand_ how it works,
+we _recommend_ taking the time to **learn** each item on the list.
If you are `new` to Phoenix, Elixir, or to Tachyons (_the UI_) @@ -53,6 +56,12 @@ which will bring you up-to-speed: + Tachyons: [github.com/dwyl/**learn-tachyons**](https://github.com/dwyl/learn-tachyons) +_Additionally_ we use Amazon Web Services (***AWS***) +Simple Storage Service (***S3***)
+for storing the issue comment _history_ with "_high availability_".
+To run `github-backup` on your `localhost` +you will need to have an AWS account and an S3 "bucket".
+(_instructions given below_). -### Set Up Checklist on `localhost` +## Set Up _Checklist_ on `localhost` -#### Install Dependencies +### Install Dependencies -To run the App on your `localhost` you will need to have **4 dependencies** _installed_: +To run the App on your `localhost`, +you will need to have **4 dependencies** _installed_: + Elixir: https://github.com/dwyl/learn-elixir#installation + PostgreSQL: https://github.com/dwyl/learn-postgresql#installation -+ Phoenix: ++ Phoenix: https://hexdocs.pm/phoenix/installation.html -+ ngrok: https://ngrok.com/download -(_so you can share your `localhost` url with GitHub_) ++ **ngrok**: https://github.com/dwyl/learn-ngrok#1-download--install
+(_so you can share the app on your `localhost` with GitHub via `public` URL_) Once all the necessary dependencies are installed, we can move on. -(_if you get stuck, please let us know_!) +(_if you get stuck, + [please let us know](https://github.com/dwyl/github-backup/issues)_!) -#### Clone the App +### Clone the App -Run the following the command to `clone` the app: +Run the following the command to `clone` the app from GitHub: ```sh -git clone git@github.com:dwyl/github-backup.git && cd github-backup +git clone https://github.com/dwyl/github-backup.git && cd github-backup ``` -#### Required Environment Variables +### Required Environment Variables + +To run the project on your `localhost`, +you will need to have the following Environment Variables defined. +> _**Note**: if you are `new` to "Environment Variables",
+**please read**_: +[github.com/dwyl/**learn-environment-variables**](https://github.com/dwyl/learn-environment-variables) ++ `SECRET_KEY_BASE` - a 64bit string used by Phoenix for security +(_to sign cookies and CSRF tokens_). See below for how to _generate_ yours. ++ `PRIVATE_KEY` - The key for your GitHub App. +_See below for how to set this up_. ++ `GITHUB_APP_ID` - The unique `id` of your GitHub App. _See below_. +#### Copy The `.env_sample` File -#### Install Elixir (_Application-specific_) Dependencies +The _easy_ way manage your Environment Variables _locally_ +is to have a `.env` file in the _root_ of the project. + +_Copy_ the _sample_ one and update the variables: ```sh -mix deps.get +cp .env_sample .env ``` +Now update the _values_ to the _real_ ones for your App.
+You will need to _register_ a GitHub app, +so that is what we will do _below_! + +> _Don't worry, your sensitive variables are **safe** +as the `.env` file is "ignored" in the_ +[`.gitignore` file](https://github.com/dwyl/github-backup/blob/778d1e311a37564721989e842412880994eb2b5d/.gitignore#L28) + -#### Create a GitHub Application +#### Generate the `SECRET_KEY_BASE` + +Run the following command +to generate a new `SECRET_KEY_BASE`: + +```sh +mix phx.gen.secret +``` +_copy-paste_ the _output_ (64bit `String`) +into your `.env` file after the "equals sign" on the line that reads: +```yml +export SECRET_KEY_BASE=run:mix_phx.gen.secret... +``` + +> _**Note**: This method only adds the environment variables **locally** +and **temporarily**
+so you need to start your server in the **same terminal** +where you ran the `source` command_. + + +### Create a GitHub Application The role of the Github application is to send **notifications** when **events** occur in your repositories.
For example you can get be notified when **new issues** or **pull requests** are ***opened***. -- Access the [new application settings page](https://github.com/settings/apps/new) on your Github account: - Settings -> Developer settings -> Github Apps -> New Github App +#### 1. Access the New Application Settings Page - ![new Github app](https://user-images.githubusercontent.com/6057298/34667319-75439af0-f460-11e7-8ae5-a9f52944b364.png) +While logged into your GitHub Account, visit: +https://github.com/settings/apps/new +Or _navigate_ the following menus: +``` +Settings -> Developer settings -> Github Apps -> New Github App +``` +![register-new-github-app](https://user-images.githubusercontent.com/194400/37253027-f20b4fbc-2523-11e8-9154-11c7f6644bb6.png) -- Github App name: The name of the app; must be unique, so can't be "gh-backup" as that's taken! +- Github App name: The name of the app; must be unique, +so can't be "gh-backup" as that's taken! - Descriptions: A short description of the app; "My backup app" - Homepage URL: The website of the app: "https://dwyl.com/" -- User authorization callback URL: Redirect url after user authentication e.g."http://localhost:4000/auth/github/callback". This is not needed for backup so this field can be left empty. -- Setup URL: Redirect the user to this url after installation, not needed for `github-backup` -- Webhook URL: URL where post requests from Github are sent to. The endpoint is ```/event/new```, however Github won't be able to send requests to ```http://localhost:4000/event/new``` as this url is only accessible by your own machine. To expose publicly your `localhost` server you can use `ngrok`. **Remember to update this value after you have a running server on your machine!** +- User authorization callback URL: Redirect url after user authentication e.g."http://localhost:4000/auth/github/callback". +This is not needed for backup so this field can be left empty. +- Setup URL: Redirect the user to this url after installation, +not needed for `github-backup`. +- Webhook URL: the URL where HTTP `POST` requests from Github are sent. +The endpoint is ```/event/new```, +however Github won't be able to send requests +to ```http://localhost:4000/event/new``` +as this url is only accessible on your `localhost`. +To allow GitHub to access your `localhost` server you can use `ngrok`. +**Remember to update these value after you have a running server on your machine!** + - Install [ngrok](https://ngrok.com). If you have homebrew, you can do this by running `brew cask install ngrok` +Then in your terminal enter `ngrok http 4000` +to generate an SSH "tunnel" between your localhost:4000 and ngrok.io. +Copy the ngrok url that appears in your terminal +to the Github app configuration; e.g: "http://bf541ce5.ngrok.io/event/new" - Then in your terminal enter `ngrok http 4000` to generate an SSH between your localhost:4000 and ngrok.io. Copy the ngrok url that appears in your terminal to the Github app configuration; "http://bf541ce5.ngrok.io/event/new" +> _NOTE: you will need to update the webhook URL everytime you disconnect/connect to ngrok because a different URL is generated everytime._ - > _NOTE: you will need to update the webhook URL everytime you disconnect/connect to ngrok because a different URL is generated everytime._ +You can read more about webhooks and ngrok at https://developer.github.com/webhooks/configuring/ - You can read more about webhooks and ngrok at https://developer.github.com/webhooks/configuring/ -- Define the access rights for the application on the permmission section. **Change "issues" to "Read only"** +#### 2. Set the Necessary Permissions -![image](https://user-images.githubusercontent.com/16775804/36432698-b50c54f6-1652-11e8-8330-513c06150d05.png) + - Define the access rights for the application on the permmission section. **Change "issues" to "Read only"** + ![image](https://user-images.githubusercontent.com/16775804/36432698-b50c54f6-1652-11e8-8330-513c06150d05.png) -- Select which events ```dwylbot``` will be notified on. **Check "issues" and "issue comment"** - ![image](https://user-images.githubusercontent.com/16775804/36432733-d4901592-1652-11e8-841f-06e7b4bf9b4c.png) + - Select which events ```dwylbot``` will be notified on. **Check "issues" and "issue comment"** -- Webhook secret: This token can be used to make sure that the requests received by `github-backup` are from Github; `github-backup` doesn't use this token at the moment so you can keep this field empty (see https://developer.github.com/webhooks/securing/) + ![image](https://user-images.githubusercontent.com/16775804/36432733-d4901592-1652-11e8-841f-06e7b4bf9b4c.png) -- You can decide to allow other users to install the Github Application or to limit the app on your account only: - ![Github app install scope](https://user-images.githubusercontent.com/6057298/34677046-cf874e96-f486-11e7-9f60-912f3ec2809b.png) + - Webhook secret: This token can be used to make sure that the requests received by `github-backup` are from Github; `github-backup` doesn't use this token at the moment so you can keep this field empty (see https://developer.github.com/webhooks/securing/) - You can now click on "Create Github App"! + - You can decide to allow other users to install the Github Application or to limit the app on your account only: + ![Github app install scope](https://user-images.githubusercontent.com/6057298/34677046-cf874e96-f486-11e7-9f60-912f3ec2809b.png) -- Create a private key: This key is used to identify specific `github-backup` installations + You can now click on "Create Github App"! - ![Github App private key](https://user-images.githubusercontent.com/6057298/34678365-d9d73dd0-f48a-11e7-8d1b-cfbfa11bbcc9.png) + - Create a private key: This key is used to identify specific `github-backup` installations - The downloaded file contains the private key. - Copy this key in your environment variables, see the next section. + ![Github App private key](https://user-images.githubusercontent.com/6057298/34678365-d9d73dd0-f48a-11e7-8d1b-cfbfa11bbcc9.png) + The downloaded file contains the private key. + Copy this key in your environment variables, see the next section. -You can also read the Github guide on how to create a new Github App at https://developer.github.com/apps/building-github-apps/creating-a-github-app/ -#### Run a `github-backup` server + You can also read the Github guide on how to create a new Github App at https://developer.github.com/apps/building-github-apps/creating-a-github-app/ -The `github-backup` server will receive events from Github and then save the details or edits to an external database. + #### Run a `github-backup` server -- Clone the repository _to your personal computer_: - ``` - git clone git@github.com:dwyl/github-backup.git && cd github-backup - ``` -- Define the local environment variables: + The `github-backup` server will receive events from Github + and then save the details or edits to an external database. - > If you are new to "Environment Variables", please read: - [github.com/dwyl/**learn-environment-variables**](https://github.com/dwyl/learn-environment-variables) - To run the application on your localhost (_personal computer_) - create an `.env` file where you can define your environment variables. +### S3 Bucket - `github-backup/.env`: - ``` - # SECRET_KEY_BASE is required for Auth Cookie: - export SECRET_KEY_BASE=MustBeA64ByteStringProbablyBestToGenerateUsingCryptoOrJustUseThisWithSomeRandomDigitsOnTheEnd1234567890 - # PRIVATE_KEY should be generated in your github app settings - export PRIVATE_KEY=YOUR_PRIVATE_KEY - # Your GITHUB_APP_ID is found in the settings of your github app under General > About - export GITHUB_APP_ID=YOUR_GITHUB_APP_ID - ``` - You can generate a new secret key base with: - ```sh - mix phx.gen.secret - ``` - Then execute the command ```source .env``` which will create your environment variables - > _**Note**: This method only adds the environment variables **locally** - and **temporarily**
- so you need to start your server in the **same terminal** - where you ran the `source` command_. -- Install dependencies: +#### "Source" the `.env` File - ``` - mix deps.get && npm install - ``` +Now that you have defined all the necessary +environment variables in your `.env` file,
-- Confirm everything is working by running the tests: +Execute the command following command in your terminal: +```sh +source .env +``` + +### Install Elixir (_Application-specific_) Dependencies + +Now that you have the environment variables defined, +you can install the _elixir_ (_application-specific_) dependencies: + +```sh +mix deps.get && npm install +``` + +#### Run the Tests! (_To check it works!_) - ``` - mix test - ``` +Confirm everything is working by running the tests: -- Create the Database (_if it does not already exist_) +```sh +mix test +``` - ``` - mix ecto.create && mix ecto.migrate - ``` -- Run the Server +### Create the Database + +In your terminal, run the following command: +```sh +mix ecto.create && mix ecto.migrate +``` + +> _**Note**: your PostgreSQL server will need to be running for this to work_. + + +### Run the Phoenix (Application) Server + +``` +mix phx.server +``` - ``` - mix phx.server - ``` +- Now that your server is running you can update the +`webhook url` in your app config: -- Now that your server is running you can update the `webhook url` in your app config: - - run ```ngrok http 4000``` in a new terminal +- run ```ngrok http 4000``` in a `new` terminal window. ![ngrok](https://user-images.githubusercontent.com/6057298/34685179-73b6d71c-f49f-11e7-8dab-abfc64c9e938.png) - copy and save the url into your Github App config with ```/event/new``` for endpoint From 4f2741e29792ffc141c83666fb861b1605bf9a62 Mon Sep 17 00:00:00 2001 From: nelsonic Date: Sun, 11 Mar 2018 20:43:02 +0000 Subject: [PATCH 08/23] adds numbering for steps in creating a new github application for #35 --- .env_sample | 6 ++-- README.md | 88 +++++++++++++++++++++++++++++++---------------------- 2 files changed, 55 insertions(+), 39 deletions(-) diff --git a/.env_sample b/.env_sample index 9eb57b2..facbfc4 100644 --- a/.env_sample +++ b/.env_sample @@ -1,8 +1,8 @@ -# SECRET_KEY_BASE is required for Auth Cookie: -export SECRET_KEY_BASE=run:mix_phx.gen.secret-in-your-terminal-and-paste-result-here - # PRIVATE_KEY should be generated in your GitHub App settings export PRIVATE_KEY=YOUR_PRIVATE_KEY # GITHUB_APP_ID is found in the settings of your GitHub app under General > About export GITHUB_APP_ID=YOUR_GITHUB_APP_ID + +# SECRET_KEY_BASE is required for Auth Cookie: +export SECRET_KEY_BASE=run:mix_phx.gen.secret-in-your-terminal-and-paste-result-here diff --git a/README.md b/README.md index 1f639c4..b9e8a7a 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ -# github-backup +# :octocat: :back: :up: GitHub Backup! -An App that helps you **Backup** your **GitHub Issues** +An App that helps you **backup** your **GitHub Issues** `so that` you can still **work** when (you/they are) ***offline***. @@ -71,6 +71,9 @@ https://github.com/dwyl/github-backup/issues/55 ## Set Up _Checklist_ on `localhost` +This will take approximately **10 minutes** to complete. +(_provided you already have a GitHub and AWS account_) + ### Install Dependencies To run the App on your `localhost`, @@ -106,11 +109,11 @@ you will need to have the following Environment Variables defined. **please read**_: [github.com/dwyl/**learn-environment-variables**](https://github.com/dwyl/learn-environment-variables) -+ `SECRET_KEY_BASE` - a 64bit string used by Phoenix for security -(_to sign cookies and CSRF tokens_). See below for how to _generate_ yours. + `PRIVATE_KEY` - The key for your GitHub App. _See below for how to set this up_. + `GITHUB_APP_ID` - The unique `id` of your GitHub App. _See below_. ++ `SECRET_KEY_BASE` - a 64bit string used by Phoenix for security +(_to sign cookies and CSRF tokens_). See below for how to _generate_ yours. #### Copy The `.env_sample` File @@ -133,25 +136,6 @@ as the `.env` file is "ignored" in the_ [`.gitignore` file](https://github.com/dwyl/github-backup/blob/778d1e311a37564721989e842412880994eb2b5d/.gitignore#L28) -#### Generate the `SECRET_KEY_BASE` - -Run the following command -to generate a new `SECRET_KEY_BASE`: - -```sh -mix phx.gen.secret -``` -_copy-paste_ the _output_ (64bit `String`) -into your `.env` file after the "equals sign" on the line that reads: -```yml -export SECRET_KEY_BASE=run:mix_phx.gen.secret... -``` - -> _**Note**: This method only adds the environment variables **locally** -and **temporarily**
-so you need to start your server in the **same terminal** -where you ran the `source` command_. - ### Create a GitHub Application @@ -163,43 +147,53 @@ when **new issues** or **pull requests** are ***opened***. #### 1. Access the New Application Settings Page While logged into your GitHub Account, visit: -https://github.com/settings/apps/new +https://github.com/settings/apps/new
Or _navigate_ the following menus: ``` Settings -> Developer settings -> Github Apps -> New Github App ``` -![register-new-github-app](https://user-images.githubusercontent.com/194400/37253027-f20b4fbc-2523-11e8-9154-11c7f6644bb6.png) +![github-backup-create-new-app-numbered](https://user-images.githubusercontent.com/194400/37258239-b5101ba8-256c-11e8-90e1-e2ca2a0ac20c.png) + + -- Github App name: The name of the app; must be unique, +1. **Github App name**: The name of the app; must be unique, so can't be "gh-backup" as that's taken! -- Descriptions: A short description of the app; "My backup app" -- Homepage URL: The website of the app: "https://dwyl.com/" -- User authorization callback URL: Redirect url after user authentication e.g."http://localhost:4000/auth/github/callback". +2. **Descriptions**: A short description of the app; "My backup app" +3. **Homepage URL**: The website of the app. e.g: "https://gitbu.io" + (_yours will be different_) +4. **User authorization callback URL**: Redirect url after user authentication e.g."http://localhost:4000/auth/github/callback". This is not needed for backup so this field can be left empty. -- Setup URL: Redirect the user to this url after installation, +5. **Setup URL** (optional): Redirect the user to this url after installation, not needed for `github-backup`. -- Webhook URL: the URL where HTTP `POST` requests from Github are sent. -The endpoint is ```/event/new```, +6. **Webhook URL**: the URL where HTTP `POST` requests from Github are sent. +The endpoint in the `github-backup` App is `/event/new`, however Github won't be able to send requests to ```http://localhost:4000/event/new``` as this url is only accessible on your `localhost`. To allow GitHub to access your `localhost` server you can use `ngrok`. **Remember to update these value after you have a running server on your machine!** - +7. **Webhook secret** (optional) - leave this blank for now. +see: https://github.com/dwyl/github-backup/issues/76 +(_please let us know your thoughts!_) Then in your terminal enter `ngrok http 4000` to generate an SSH "tunnel" between your localhost:4000 and ngrok.io. Copy the ngrok url that appears in your terminal to the Github app configuration; e.g: "http://bf541ce5.ngrok.io/event/new" -> _NOTE: you will need to update the webhook URL everytime you disconnect/connect to ngrok because a different URL is generated everytime._ +> _NOTE: you will need to update the webhook URL each time you +disconnect/connect to ngrok because a different URL is generated every time._ -You can read more about webhooks and ngrok at https://developer.github.com/webhooks/configuring/ +You can read more about webhooks and `ngrok` at: +https://developer.github.com/webhooks/configuring #### 2. Set the Necessary Permissions - - Define the access rights for the application on the permmission section. **Change "issues" to "Read only"** +Still on the same page, just scrolled down below the initial setup. + + - Define the access rights for the application on the permission section. **Change "issues" to "Read only"** ![image](https://user-images.githubusercontent.com/16775804/36432698-b50c54f6-1652-11e8-8330-513c06150d05.png) @@ -237,6 +231,22 @@ You can read more about webhooks and ngrok at https://developer.github.com/webho +#### Generate the `SECRET_KEY_BASE` + +Run the following command +to generate a new `SECRET_KEY_BASE`: + +```sh +mix phx.gen.secret +``` +_copy-paste_ the _output_ (64bit `String`) +into your `.env` file after the "equals sign" on the line that reads: +```yml +export SECRET_KEY_BASE=run:mix_phx.gen.secret... +``` + + + #### "Source" the `.env` File Now that you have defined all the necessary @@ -248,6 +258,12 @@ Execute the command following command in your terminal: source .env ``` +> _**Note**: This method only adds the environment variables **locally** +and **temporarily**
+so you need to start your server in the **same terminal** +where you ran the `source` command_. + + ### Install Elixir (_Application-specific_) Dependencies Now that you have the environment variables defined, From d05bc691cae2484934409ccf7d1b4cec29360a78 Mon Sep 17 00:00:00 2001 From: nelsonic Date: Mon, 12 Mar 2018 19:18:03 +0000 Subject: [PATCH 09/23] adds APP_HOST to .env_sample for #80 #77 #14 --- .env_sample | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.env_sample b/.env_sample index facbfc4..15ec2cf 100644 --- a/.env_sample +++ b/.env_sample @@ -1,3 +1,6 @@ +# APP_HOST is the host of your app, if you deploy the app change to your domain. +export APP_HOST=localhost + # PRIVATE_KEY should be generated in your GitHub App settings export PRIVATE_KEY=YOUR_PRIVATE_KEY From cc286fc2e9ac7cb39bc01944bdbd75216d75150d Mon Sep 17 00:00:00 2001 From: nelsonic Date: Mon, 12 Mar 2018 19:38:04 +0000 Subject: [PATCH 10/23] adds GITHUB_APP_NAME to both Readme env var list and .env_sample file for #80 #77 #14 --- .env_sample | 3 +++ README.md | 2 ++ 2 files changed, 5 insertions(+) diff --git a/.env_sample b/.env_sample index 15ec2cf..421e42a 100644 --- a/.env_sample +++ b/.env_sample @@ -7,5 +7,8 @@ export PRIVATE_KEY=YOUR_PRIVATE_KEY # GITHUB_APP_ID is found in the settings of your GitHub app under General > About export GITHUB_APP_ID=YOUR_GITHUB_APP_ID +# GITHUB_APP_NAME what you called your app. +export GITHUB_APP_NAME=NAME_OF_THE_GITHUB_APP + # SECRET_KEY_BASE is required for Auth Cookie: export SECRET_KEY_BASE=run:mix_phx.gen.secret-in-your-terminal-and-paste-result-here diff --git a/README.md b/README.md index b9e8a7a..be9d894 100644 --- a/README.md +++ b/README.md @@ -109,9 +109,11 @@ you will need to have the following Environment Variables defined. **please read**_: [github.com/dwyl/**learn-environment-variables**](https://github.com/dwyl/learn-environment-variables) ++ `APP_HOST` - the hostname for your app e.g: `localhost` or `gitbu.io`. + `PRIVATE_KEY` - The key for your GitHub App. _See below for how to set this up_. + `GITHUB_APP_ID` - The unique `id` of your GitHub App. _See below_. ++ `GITHUB_APP_NAME` - the name of your GitHub App. _See below_ (_Step 1_). + `SECRET_KEY_BASE` - a 64bit string used by Phoenix for security (_to sign cookies and CSRF tokens_). See below for how to _generate_ yours. From 5adab8b6f931364cbfa4c32d755ddb9cb4579806 Mon Sep 17 00:00:00 2001 From: nelsonic Date: Fri, 23 Mar 2018 15:35:35 +0000 Subject: [PATCH 11/23] adds link to https://github.com/dwyl/learn-ngrok --- .env_sample | 4 ++-- README.md | 15 ++++++++------- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/.env_sample b/.env_sample index 421e42a..94b6a11 100644 --- a/.env_sample +++ b/.env_sample @@ -4,11 +4,11 @@ export APP_HOST=localhost # PRIVATE_KEY should be generated in your GitHub App settings export PRIVATE_KEY=YOUR_PRIVATE_KEY -# GITHUB_APP_ID is found in the settings of your GitHub app under General > About +# GITHUB_APP_ID is found in settings of your GitHub app under General > About export GITHUB_APP_ID=YOUR_GITHUB_APP_ID # GITHUB_APP_NAME what you called your app. export GITHUB_APP_NAME=NAME_OF_THE_GITHUB_APP # SECRET_KEY_BASE is required for Auth Cookie: -export SECRET_KEY_BASE=run:mix_phx.gen.secret-in-your-terminal-and-paste-result-here +export SECRET_KEY_BASE=run:mix_phx.gen.secret-in-terminal-and-paste-result-here diff --git a/README.md b/README.md index be9d894..c72f06a 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# :octocat: :back: :up: GitHub Backup! +# GitHub Backup! An App that helps you **backup** your **GitHub Issues** `so that` you can still **work** when (you/they are) ***offline***. @@ -10,9 +10,7 @@ An App that helps you **backup** your **GitHub Issues** [![Inline docs](http://inch-ci.org/github/dwyl/github-backup.svg?style=flat-square)](http://inch-ci.org/github/dwyl/github-backup) [![codecov.io](https://img.shields.io/codecov/c/github/dwyl/github-backup/master.svg?style=flat-square)](http://codecov.io/github/dwyl/github-backup?branch=master) [![Deps Status](https://beta.hexfaktor.org/badge/all/github/dwyl/github-backup.svg?style=flat-square)](https://beta.hexfaktor.org/github/dwyl/github-backup) -
@@ -28,9 +26,10 @@ I `need` a backup of GitHub issues
GitHub (Issue) Backup is an App that lets you (_unsurprisingly_): -1. **Store a backup **of the content of the **GitHub issue(s)** +1. **Store a backup** of the content of the **GitHub issue(s)** for a given project/repo. 2. Track the changes/edits made to issue description and comments. +see: [dear-github/issues/129](https://github.com/dear-github/dear-github/issues/129) 3. View all your issues when you or GitHub are ["offline"](https://github.com/dwyl/github-reference/issues/15) @@ -45,7 +44,7 @@ if you want to _understand_ how it works,
we _recommend_ taking the time to **learn** each item on the list.
If you are `new` to Phoenix, Elixir, -or to Tachyons (_the UI_) +Tachyons (_the UI_) or ngrok,
we have a "_beginner tutorials_" which will bring you up-to-speed: @@ -55,10 +54,12 @@ which will bring you up-to-speed: [github.com/dwyl/**learn-phoenix-framework**](https://github.com/dwyl/learn-phoenix-framework) + Tachyons: [github.com/dwyl/**learn-tachyons**](https://github.com/dwyl/learn-tachyons) ++ ngrok: [github.com/dwyl/**learn-ngrok**](https://github.com/dwyl/learn-ngrok) _Additionally_ we use Amazon Web Services (***AWS***) Simple Storage Service (***S3***)
-for storing the issue comment _history_ with "_high availability_".
+for storing the issue comment _history_ +with "_high availability_" and reliability.
To run `github-backup` on your `localhost` you will need to have an AWS account and an S3 "bucket".
(_instructions given below_). @@ -71,7 +72,7 @@ https://github.com/dwyl/github-backup/issues/55 ## Set Up _Checklist_ on `localhost` -This will take approximately **10 minutes** to complete. +This will take approximately **15 minutes** to complete. (_provided you already have a GitHub and AWS account_) ### Install Dependencies From 97a2bddb1a893370982501dfae7e1963be4b48bd Mon Sep 17 00:00:00 2001 From: nelsonic Date: Fri, 23 Mar 2018 20:13:20 +0000 Subject: [PATCH 12/23] adds keytoenvar.sh script to add PRIVATE_KEY (RSA Private Key) environment variable see: https://github.com/dwyl/learn-environment-variables/issues/17 --- keytoenvar.sh | 4 ++++ 1 file changed, 4 insertions(+) create mode 100755 keytoenvar.sh diff --git a/keytoenvar.sh b/keytoenvar.sh new file mode 100755 index 0000000..30482de --- /dev/null +++ b/keytoenvar.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +file=$2 +name=$1 +export $name="$(awk 'BEGIN{}{out=out$0"\n"}END{print out}' $file| sed 's/\n$//')" From f8bb513d52a99c2194b0cef0fc56df33812a53d2 Mon Sep 17 00:00:00 2001 From: nelsonic Date: Fri, 23 Mar 2018 20:13:59 +0000 Subject: [PATCH 13/23] adds "*.pem" to .gitignore to ensure RSA Private Keys are not committed to GitHub --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 2f3cd6a..9cfedf3 100644 --- a/.gitignore +++ b/.gitignore @@ -27,3 +27,4 @@ npm-debug.log /config/*.secret.exs .env cover +*.pem From 528aacdb2a3c1560261ed75b556cd104545ca510 Mon Sep 17 00:00:00 2001 From: nelsonic Date: Fri, 23 Mar 2018 20:52:34 +0000 Subject: [PATCH 14/23] [WiP] adds instructions to export PRIVATE_KEY (RSA Key) environment variables ... --- README.md | 83 +++++++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 66 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index 8896fb1..4f7113b 100644 --- a/README.md +++ b/README.md @@ -147,6 +147,10 @@ send **notifications** when **events** occur in your repositories.
For example you can get be notified when **new issues** or **pull requests** are ***opened***. +> _**Note**: we have **simplified** +the steps in the "official" Github guide +to create a new Github App_: https://developer.github.com/apps/building-github-apps/creating-a-github-app + #### 1. Access the New Application Settings Page While logged into your GitHub Account, visit: @@ -173,13 +177,13 @@ not needed for `github-backup`. 6. **Webhook URL**: the URL where HTTP `POST` requests from Github are sent. The endpoint in the `github-backup` App is `/event/new`, however Github won't be able to send requests -to ```http://localhost:4000/event/new``` +to `http://localhost:4000/event/new` as this url is only accessible on your `localhost`. To allow GitHub to access your `localhost` server you can use `ngrok`. **Remember to update these value after you have a running server on your machine!** 7. **Webhook secret** (optional) - leave this blank for now. -see: https://github.com/dwyl/github-backup/issues/76 -(_please let us know your thoughts!_) +see: [github-backup/issues/76](https://github.com/dwyl/github-backup/issues/76) +(_please let us know your thoughts on how/when we should prioritise this!_) Then in your terminal enter `ngrok http 4000` to generate an SSH "tunnel" between your localhost:4000 and ngrok.io. @@ -194,33 +198,78 @@ https://developer.github.com/webhooks/configuring #### 2. Set the Necessary Permissions -Still on the same page, just scrolled down below the initial setup. +Still on the same page, scroll down below the initial setup, +to the ***Permissions*** section: + +![github-backup-permissions](https://user-images.githubusercontent.com/194400/37843463-e1498be2-2ebc-11e8-905e-4c12e359e080.png) + +1. Define the access rights for the application on the permission section. +**Change "issues" to "Read & write"** +(_this is required to update "meta table" which links to the "issue history"_) + + +#### 3. Subscribe to Events + +Scroll down to the "***Subscribe to events***" section +and check the boxes for ***Issue comment*** and ***Issues***: +![github-backup-subscribe-to-events](https://user-images.githubusercontent.com/194400/37843759-c0116002-2ebd-11e8-95a5-242c0238e115.png) + + +#### 4. Where can this GitHub App be installed? - - Define the access rights for the application on the permission section. **Change "issues" to "Read only"** +Scroll down to the "***Where can this GitHub App be installed?***" section. - ![image](https://user-images.githubusercontent.com/16775804/36432698-b50c54f6-1652-11e8-8330-513c06150d05.png) +![github-backup-where-can-be-installed](https://user-images.githubusercontent.com/194400/37844500-dee11980-2ebf-11e8-904b-94a34559c2e2.png) +In _our_ case we want to be able to use GitHub Backup for a number of projects +in different organisations, so we selected ***Any account***. +but if you only want to backup your _personal_ projects, +then select ***Only on this account***. - - Select which events ```dwylbot``` will be notified on. **Check "issues" and "issue comment"** +#### 5. Click on the `Create GitHub App` Button! - ![image](https://user-images.githubusercontent.com/16775804/36432733-d4901592-1652-11e8-841f-06e7b4bf9b4c.png) +_Finally_, with everything configured, click on "Create Github App"! - - Webhook secret: This token can be used to make sure that the requests received by `github-backup` are from Github; `github-backup` doesn't use this token at the moment so you can keep this field empty (see https://developer.github.com/webhooks/securing/) +![create-github-app](https://user-images.githubusercontent.com/194400/37844737-a0466b84-2ec0-11e8-8ef3-dff6e2f6162a.png) - - You can decide to allow other users to install the Github Application or to limit the app on your account only: - ![Github app install scope](https://user-images.githubusercontent.com/6057298/34677046-cf874e96-f486-11e7-9f60-912f3ec2809b.png) +You should see a message confirming that your app was created: +![github-backup-create-private-key](https://user-images.githubusercontent.com/194400/37846890-fa8f9bbe-2ec6-11e8-9324-155f5f444f56.png) - You can now click on "Create Github App"! +(_obviously your app will will have a different name in the url/path..._) - - Create a private key: This key is used to identify specific `github-backup` installations +Click on the _link_ to "***generate a private key***". - ![Github App private key](https://user-images.githubusercontent.com/6057298/34678365-d9d73dd0-f48a-11e7-8d1b-cfbfa11bbcc9.png) +![github-backup-private-key](https://user-images.githubusercontent.com/194400/37848805-53bb653c-2ecd-11e8-9a80-8d328ad9aadf.png) + +When you click the `Generate private key` button it should _download_ +a file containing your private key. + +![github-backup-pk-download](https://user-images.githubusercontent.com/194400/37848886-979979f6-2ecd-11e8-98a3-3b69dcf8dae6.png) + +Open the file in your text editor, e.g: +![private-key](https://user-images.githubusercontent.com/194400/37849176-94cb965e-2ece-11e8-8771-02538aa662d9.png) + +> _**Don't worry**, this is **not** our "**real**" private key ... +it's just for demonstration purposes.
+but this is what your RSA private key will +look like, a block of random characters ..._ + + +The downloaded file contains the private key. +Copy this key in your `.env` file in the `PRIVATE_KEY`: +e.g:
+```text +# PRIVATE_KEY should be generated in your GitHub App settings +export PRIVATE_KEY=-----BEGIN RSA PRIVATE KEY----- +MIIEpAIBAAKCAQEA04up8hoqzS1+ +... +l48DlnUtMdMrWvBlRFPzU+hU9wDhb3F0CATQdvYo2mhzyUs8B1ZSQz2Vy== +-----END RSA PRIVATE KEY----- +``` - The downloaded file contains the private key. - Copy this key in your environment variables, see the next section. +source keytoenvar.sh PRIVATE_KEY ./gitbu.2018-03-23.private-key.pem - You can also read the Github guide on how to create a new Github App at https://developer.github.com/apps/building-github-apps/creating-a-github-app/ #### Run a `github-backup` server From a1a7d271230f194e53935bb28565a6dc837a9980 Mon Sep 17 00:00:00 2001 From: nelsonic Date: Sat, 24 Mar 2018 16:36:24 +0000 Subject: [PATCH 15/23] adds AWS/S3 environment varialbes to list of required & .env_sample for #35 --- .env_sample | 5 +++++ README.md | 9 +++++++-- config/config.exs | 18 ------------------ 3 files changed, 12 insertions(+), 20 deletions(-) diff --git a/.env_sample b/.env_sample index 94b6a11..d52ade0 100644 --- a/.env_sample +++ b/.env_sample @@ -12,3 +12,8 @@ export GITHUB_APP_NAME=NAME_OF_THE_GITHUB_APP # SECRET_KEY_BASE is required for Auth Cookie: export SECRET_KEY_BASE=run:mix_phx.gen.secret-in-terminal-and-paste-result-here + +# AWS: +export AWS_ACCESS_KEY_ID=aws-access-key-here +export AWS_SECRET_ACCESS_KEY=super-secret-key +export S3_BUCKET_NAME=your-bucket-name diff --git a/README.md b/README.md index 4f7113b..72b92f5 100644 --- a/README.md +++ b/README.md @@ -111,7 +111,7 @@ you will need to have the following Environment Variables defined. [github.com/dwyl/**learn-environment-variables**](https://github.com/dwyl/learn-environment-variables) + `APP_HOST` - the hostname for your app e.g: `localhost` or `gitbu.io`. -+ `PRIVATE_KEY` - The key for your GitHub App. ++ `PRIVATE_KEY` - The RSA private key for your GitHub App. _See below for how to set this up_. + `GITHUB_APP_ID` - The unique `id` of your GitHub App. _See below_. + `GITHUB_APP_NAME` - the name of your GitHub App. _See below_ (_Step 1_). @@ -267,10 +267,15 @@ l48DlnUtMdMrWvBlRFPzU+hU9wDhb3F0CATQdvYo2mhzyUs8B1ZSQz2Vy== -----END RSA PRIVATE KEY----- ``` - source keytoenvar.sh PRIVATE_KEY ./gitbu.2018-03-23.private-key.pem +In the "About" section you will find the `ID` of your app e.g: 103 +this is the number that needs to be set for `GITHUB_APP_ID` in your `.env` file. + +![app-about-id](https://user-images.githubusercontent.com/194400/37863366-004a4f02-2f55-11e8-97eb-7d9f8b74ba66.png) + + #### Run a `github-backup` server The `github-backup` server will receive events from Github diff --git a/config/config.exs b/config/config.exs index 4878f5e..c783248 100644 --- a/config/config.exs +++ b/config/config.exs @@ -22,26 +22,8 @@ config :logger, :console, format: "$time $metadata[$level] $message\n", metadata: [:request_id] - -# config :ex_aws, -# access_key_id: [{:system, System.get_env("AWS_ACCESS_KEY_ID")}, :instance_role], -# secret_access_key: [{:system, System.get_env("AWS_SECRET_ACCESS_KEY")}, :instance_role], -# region: "eu-west-2" - -# Configure :ex_aws - Taken from JC tutorial -# Or take the version from ex-aws? with instance_role -config :ex_aws, - access_key_id: System.get_env("AWS_ACCESS_KEY_ID"), - secret_access_key: System.get_env("AWS_SECRET_ACCESS_KEY"), - s3: [ - scheme: "https://", - host: "dwyl-github-backup.s3.amazonaws.com", - region: "eu-west-2" - ] - config :app, :github_app_name, Map.fetch!(System.get_env(), "GITHUB_APP_NAME") - # Import environment specific config. This must remain at the bottom # of this file so it overrides the configuration defined above. import_config "#{Mix.env}.exs" From e403e3ff4e2b66cee35c39e3335683c69d4150fe Mon Sep 17 00:00:00 2001 From: nelsonic Date: Sun, 25 Mar 2018 17:30:03 +0100 Subject: [PATCH 16/23] fix failing test :four_leaf_clover: --- config/dev.exs | 2 -- config/prod.exs | 2 -- lib/app_web/controllers/aws/s3.ex | 2 -- lib/app_web/controllers/event_controller.ex | 2 +- lib/app_web/controllers/page_controller.ex | 3 -- lib/app_web/controllers/upload_controller.ex | 37 -------------------- mix.lock | 2 -- 7 files changed, 1 insertion(+), 49 deletions(-) delete mode 100644 lib/app_web/controllers/upload_controller.ex diff --git a/config/dev.exs b/config/dev.exs index e11a591..8c8ece0 100644 --- a/config/dev.exs +++ b/config/dev.exs @@ -58,6 +58,4 @@ config :app, App.Repo, pool_size: 10 config :app, :github_api, AppWeb.GithubAPI.HTTPClient - config :app, :s3_api, AppWeb.AWS.S3 - diff --git a/config/prod.exs b/config/prod.exs index 4f989b2..8bd39d8 100644 --- a/config/prod.exs +++ b/config/prod.exs @@ -26,10 +26,8 @@ config :app, App.Repo, ssl: true config :app, :github_api, AppWeb.GithubAPI.HTTPClient - config :app, :s3_api, AppWeb.AWS.S3 - # Do not print debug messages in production config :logger, level: :info diff --git a/lib/app_web/controllers/aws/s3.ex b/lib/app_web/controllers/aws/s3.ex index ff798cf..259037c 100644 --- a/lib/app_web/controllers/aws/s3.ex +++ b/lib/app_web/controllers/aws/s3.ex @@ -3,7 +3,6 @@ defmodule AppWeb.AWS.S3 do wrapper functions for saving data to aws s3 """ alias ExAws.S3 - @s3_bucket System.get_env("S3_BUCKET_NAME") @s3_region "eu-west-2" @@ -18,5 +17,4 @@ defmodule AppWeb.AWS.S3 do |> S3.get_object("#{issue_id}.json") |> ExAws.request(region: @s3_region) end - end diff --git a/lib/app_web/controllers/event_controller.ex b/lib/app_web/controllers/event_controller.ex index 2eedb54..5ef1739 100644 --- a/lib/app_web/controllers/event_controller.ex +++ b/lib/app_web/controllers/event_controller.ex @@ -11,4 +11,4 @@ defmodule AppWeb.EventController do x_github_event, github_webhook_action, conn, payload ) end -end \ No newline at end of file +end diff --git a/lib/app_web/controllers/page_controller.ex b/lib/app_web/controllers/page_controller.ex index 3ad7bc7..c71bf08 100644 --- a/lib/app_web/controllers/page_controller.ex +++ b/lib/app_web/controllers/page_controller.ex @@ -1,10 +1,7 @@ defmodule AppWeb.PageController do use AppWeb, :controller - alias AppWeb.AWS.S3 - def index(conn, _params) do - S3.get_files_bucket() render conn, "index.html" end end diff --git a/lib/app_web/controllers/upload_controller.ex b/lib/app_web/controllers/upload_controller.ex deleted file mode 100644 index d624097..0000000 --- a/lib/app_web/controllers/upload_controller.ex +++ /dev/null @@ -1,37 +0,0 @@ -# # Code from JC tutorial -# -# defmodule AppWeb.UploadController do -# use AppWeb.Web, :controller -# alias AppWeb.Upload -# -# def new(conn, _params) do -# changeset = Upload.changeset(%Upload{}) -# render conn, "new.html", changeset: changeset -# end -# -# def create(conn, %{"upload" => %{"image" => image_params} = upload_params}) do -# file_uuid = UUID.uuid4(:hex) -# image_filename = image_params.filename -# unique_filename = "#{file_uuid}-#{image_filename}" -# {:ok, image_binary} = File.read(image_params.path) -# bucket_name = System.get_env("BUCKET_NAME") -# image = -# ExAws.S3.put_object(bucket_name, unique_filename, image_binary) -# |> ExAws.request! -# -# # build the image url and add to the params to be stored -# updated_params = -# upload_params -# |> Map.update(image, image_params, fn _value -> "https://#{bucket_name}.s3.amazonaws.com/#{bucket_name}/#{unique_filename}" end) -# changeset = Upload.changeset(%Upload{}, updated_params) -# -# case Repo.insert!(changeset) do -# {:ok, upload} -> -# conn -# |> put_flash(:info, "Image uploaded successfully!") -# |> redirect(to: upload_path(conn, :new)) -# {:error, changeset} -> -# render conn, "new.html", changeset: changeset -# end -# end -# end diff --git a/mix.lock b/mix.lock index 796b065..e380bd3 100644 --- a/mix.lock +++ b/mix.lock @@ -40,9 +40,7 @@ "pre_commit": {:hex, :pre_commit, "0.2.4", "e26feaa149d55d3a6f14ca1340bd8738942d98405de389c0b3c7d48e71e62d66", [:mix], []}, "ranch": {:hex, :ranch, "1.3.2", "e4965a144dc9fbe70e5c077c65e73c57165416a901bd02ea899cfd95aa890986", [:rebar3], []}, "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.1", "28a4d65b7f59893bc2c7de786dec1e1555bd742d336043fe644ae956c3497fbe", [:make, :rebar], []}, - "sweet_xml": {:hex, :sweet_xml, "0.6.5", "dd9cde443212b505d1b5f9758feb2000e66a14d3c449f04c572f3048c66e6697", [:mix], []}, "timex": {:hex, :timex, "3.2.1", "639975eac45c4c08c2dbf7fc53033c313ff1f94fad9282af03619a3826493612", [:mix], [{:combine, "~> 0.10", [hex: :combine, optional: false]}, {:gettext, "~> 0.10", [hex: :gettext, optional: false]}, {:tzdata, "~> 0.1.8 or ~> 0.5", [hex: :tzdata, optional: false]}]}, "tzdata": {:hex, :tzdata, "0.5.16", "13424d3afc76c68ff607f2df966c0ab4f3258859bbe3c979c9ed1606135e7352", [:mix], [{:hackney, "~> 1.0", [hex: :hackney, optional: false]}]}, "unicode_util_compat": {:hex, :unicode_util_compat, "0.3.1", "a1f612a7b512638634a603c8f401892afbf99b8ce93a45041f8aaca99cadb85e", [:rebar3], []}, - "uuid": {:hex, :uuid, "1.1.8", "e22fc04499de0de3ed1116b770c7737779f226ceefa0badb3592e64d5cfb4eb9", [:mix], []}, } From d4ee53bfb1a8aadee05c7f9b42e3ad9e8dea28cd Mon Sep 17 00:00:00 2001 From: nelsonic Date: Sun, 25 Mar 2018 19:56:51 +0100 Subject: [PATCH 17/23] add fields to package.json to address NPM warnings. fix https://github.com/dwyl/github-backup/issues/113 --- assets/README.md | 3 +++ assets/package.json | 15 ++++++++++++--- 2 files changed, 15 insertions(+), 3 deletions(-) create mode 100644 assets/README.md diff --git a/assets/README.md b/assets/README.md new file mode 100644 index 0000000..53d6546 --- /dev/null +++ b/assets/README.md @@ -0,0 +1,3 @@ +Please see the _main_ +[`README.md`](https://github.com/dwyl/github-backup/blob/master/README.md) +in the root of the project. diff --git a/assets/package.json b/assets/package.json index e56314d..06faa83 100644 --- a/assets/package.json +++ b/assets/package.json @@ -1,6 +1,11 @@ { - "repository": {}, - "license": "MIT", + "name":"gitbu", + "description": "The easy way to backup GitHub issues with comment history", + "repository": { + "type": "git", + "url": "https://github.com/dwyl/github-backup.git" + }, + "license": "GPL-2.0", "scripts": { "deploy": "brunch build --production", "coverage": "mix coveralls", @@ -15,5 +20,9 @@ "brunch": "2.10.9", "clean-css-brunch": "2.10.0", "uglify-js-brunch": "2.10.0" - } + }, + "bugs": { + "url": "https://github.com/dwyl/github-backup/issues" + }, + "homepage": "https://github.com/dwyl/github-backup", } From a81aa975ce627aad5f7c6cb2302c318f104e19a8 Mon Sep 17 00:00:00 2001 From: nelsonic Date: Sun, 25 Mar 2018 23:51:57 +0100 Subject: [PATCH 18/23] adds updated screenshots of running ngrok and homepage in browser #35 --- .env_sample | 4 +- README.md | 145 +++++++++++++++++++++++++++++++------------- assets/package.json | 2 +- 3 files changed, 107 insertions(+), 44 deletions(-) diff --git a/.env_sample b/.env_sample index d52ade0..446577d 100644 --- a/.env_sample +++ b/.env_sample @@ -2,7 +2,7 @@ export APP_HOST=localhost # PRIVATE_KEY should be generated in your GitHub App settings -export PRIVATE_KEY=YOUR_PRIVATE_KEY +# see export PRIVATE_KEY=YOUR_PRIVATE_KEY # GITHUB_APP_ID is found in settings of your GitHub app under General > About export GITHUB_APP_ID=YOUR_GITHUB_APP_ID @@ -14,6 +14,6 @@ export GITHUB_APP_NAME=NAME_OF_THE_GITHUB_APP export SECRET_KEY_BASE=run:mix_phx.gen.secret-in-terminal-and-paste-result-here # AWS: -export AWS_ACCESS_KEY_ID=aws-access-key-here +export AWS_ACCESS_KEY_ID=aws-access-key export AWS_SECRET_ACCESS_KEY=super-secret-key export S3_BUCKET_NAME=your-bucket-name diff --git a/README.md b/README.md index 72b92f5..d3dcef0 100644 --- a/README.md +++ b/README.md @@ -72,19 +72,19 @@ https://github.com/dwyl/github-backup/issues/55 ## Set Up _Checklist_ on `localhost` -This will take approximately **15 minutes** to complete. +This will take approximately **15 minutes** to complete.
(_provided you already have a GitHub and AWS account_) -### Install Dependencies +### Install Dependencies To run the App on your `localhost`, you will need to have **4 dependencies** _installed_: -+ Elixir: ++ **Elixir**: https://github.com/dwyl/learn-elixir#installation -+ PostgreSQL: ++ **PostgreSQL**: https://github.com/dwyl/learn-postgresql#installation -+ Phoenix: ++ **Phoenix**: https://hexdocs.pm/phoenix/installation.html + **ngrok**: https://github.com/dwyl/learn-ngrok#1-download--install
(_so you can share the app on your `localhost` with GitHub via `public` URL_) @@ -94,7 +94,7 @@ we can move on. (_if you get stuck, [please let us know](https://github.com/dwyl/github-backup/issues)_!) -### Clone the App +### Clone the App from GitHub Run the following the command to `clone` the app from GitHub: ```sh @@ -117,6 +117,10 @@ _See below for how to set this up_. + `GITHUB_APP_NAME` - the name of your GitHub App. _See below_ (_Step 1_). + `SECRET_KEY_BASE` - a 64bit string used by Phoenix for security (_to sign cookies and CSRF tokens_). See below for how to _generate_ yours. ++ `AWS_ACCESS_KEY_ID` - your AWS access key (_get this from your AWS settings_) ++ `AWS_SECRET_ACCESS_KEY` - your AWS secrete access key ++ `S3_BUCKET_NAME` - name of the AWS S3 "bucket" where issue comments +will be stored. #### Copy The `.env_sample` File @@ -217,7 +221,7 @@ and check the boxes for ***Issue comment*** and ***Issues***: #### 4. Where can this GitHub App be installed? -Scroll down to the "***Where can this GitHub App be installed?***" section. +Scroll down to the "**_Where_ can this GitHub App be installed?**" section. ![github-backup-where-can-be-installed](https://user-images.githubusercontent.com/194400/37844500-dee11980-2ebf-11e8-904b-94a34559c2e2.png) @@ -237,6 +241,8 @@ You should see a message confirming that your app was created: (_obviously your app will will have a different name in the url/path..._) +#### 6. Generate Private Key + Click on the _link_ to "***generate a private key***". ![github-backup-private-key](https://user-images.githubusercontent.com/194400/37848805-53bb653c-2ecd-11e8-9a80-8d328ad9aadf.png) @@ -252,56 +258,67 @@ Open the file in your text editor, e.g: > _**Don't worry**, this is **not** our "**real**" private key ... it's just for demonstration purposes.
but this is what your RSA private key will -look like, a block of random characters ..._ +**look** like, a block of random characters ..._ The downloaded file contains the private key. -Copy this key in your `.env` file in the `PRIVATE_KEY`: -e.g:
-```text -# PRIVATE_KEY should be generated in your GitHub App settings -export PRIVATE_KEY=-----BEGIN RSA PRIVATE KEY----- -MIIEpAIBAAKCAQEA04up8hoqzS1+ -... -l48DlnUtMdMrWvBlRFPzU+hU9wDhb3F0CATQdvYo2mhzyUs8B1ZSQz2Vy== ------END RSA PRIVATE KEY----- -``` +Save this file in the root +of the `github-backup` repo on your `localhost`. -source keytoenvar.sh PRIVATE_KEY ./gitbu.2018-03-23.private-key.pem +For _example_ in _our_ case the GitHub app is called "**gitbu**",
+so our Private Key file starts with the app name +and the date the key was generated: +`gitbu.2018-03-23.private-key.pem` -In the "About" section you will find the `ID` of your app e.g: 103 -this is the number that needs to be set for `GITHUB_APP_ID` in your `.env` file. +> _**Don't worry**, all `.pem` (private key) files +are ignored in the `.gitignore` file so your file will stay private._ -![app-about-id](https://user-images.githubusercontent.com/194400/37863366-004a4f02-2f55-11e8-97eb-7d9f8b74ba66.png) +Once you have copied the file into your project root, run the following command: +```sh +source keytoenvar.sh PRIVATE_KEY ./gitbu.2018-03-23.private-key.pem +``` +#### 7. Copy the App Name and App `ID` from GitHub App Settings Page - #### Run a `github-backup` server +In the "**Basic information**" section of your app's settings page, +copy the GitHub App name and paste it into your `.env` file after the `=` sign +for the `GITHUB_APP_NAME` variable. - The `github-backup` server will receive events from Github - and then save the details or edits to an external database. +![github-backup-app-name](https://user-images.githubusercontent.com/194400/37878406-dbaaa2ca-3060-11e8-8c50-14860144ea79.png) +Scroll downd to the "About" section you will find the `ID` of your app e.g: 103 +this is the number that needs to be set for `GITHUB_APP_ID` in your `.env` file. + +![app-about-id](https://user-images.githubusercontent.com/194400/37863366-004a4f02-2f55-11e8-97eb-7d9f8b74ba66.png) -### S3 Bucket #### Generate the `SECRET_KEY_BASE` -Run the following command -to generate a new `SECRET_KEY_BASE`: +Run the following command to generate a new phoenix secret key: ```sh mix phx.gen.secret ``` _copy-paste_ the _output_ (64bit `String`) -into your `.env` file after the "equals sign" on the line that reads: +into your `.env` file after the "equals sign" on the line for `SECRET_KEY_BASE`: ```yml -export SECRET_KEY_BASE=run:mix_phx.gen.secret... +export SECRET_KEY_BASE=YourSecreteKeyBaseGeneratedUsing-mix_phx.gen.secret ``` +#### S3 Bucket + +In order to save the Issue Comments (_which are a variable length "rich text"_), +we save these as `.json` in an S3 bucket. If you do not _already_ have +an S3 bucket where you can store issues/comments, create one now. + +Once you have an S3 bucket, copy the name and paste it into your `.env` file +against the `S3_BUCKET_NAME` environment variable. +You will _also_ need to define the #### "Source" the `.env` File @@ -326,7 +343,7 @@ Now that you have the environment variables defined, you can install the _elixir_ (_application-specific_) dependencies: ```sh -mix deps.get && npm install +mix deps.get && cd assets npm install && cd .. ``` #### Run the Tests! (_To check it works!_) @@ -337,7 +354,6 @@ Confirm everything is working by running the tests: mix test ``` - ### Create the Database In your terminal, run the following command: @@ -345,28 +361,75 @@ In your terminal, run the following command: mix ecto.create && mix ecto.migrate ``` -> _**Note**: your PostgreSQL server will need to be running for this to work_. +> _**Note**: your **PostgreSQL** server +will need to be **running** for this to work_. + +### Run a `github-backup` (Phoenix Application) Server -### Run the Phoenix (Application) Server +The `github-backup` server will receive events from Github +and then save the details or edits to an external database. ``` mix phx.server ``` -- Now that your server is running you can update the -`webhook url` in your app config: -- run ```ngrok http 4000``` in a `new` terminal window. +### Run `ngrok` on your `localhost` + +In a New Window/Tab in your Terminal run the `ngrok` server: +```sh +ngrok http 4000 +``` + +You should see something like this: +![ngrok-terminal-running-proxy](https://user-images.githubusercontent.com/194400/37878883-a3ae8646-3067-11e8-8e1f-02683a2fd0bb.png) + +On the line that says "**forwarding**" you will see the (`public`) URL +of your app, in this case https://38e239f0.ngrok.io +(_your URL's subdomain will be a different "random" set of characters_) + +Copy the URL for the next step. (_remember to copy the `https` version_) + +### Update your Webhook URL on your GitHub App + +Now that your server is running you can update the +`webhook url` in your app config. +On the "GitHub App settings page" +Find the "Webhook URL" section: +(_the URL is currently `http://localhost:4000` ..._) + +![github-backup-webhook-url-before](https://user-images.githubusercontent.com/194400/37879083-7e929d22-306a-11e8-8a86-48afa14158e0.png) + + +_Replace+ the `http://localhost:4000` with your _unique_ `ngrok` url e.g: + +![github-backup-webhook-url-after](https://user-images.githubusercontent.com/194400/37879098-bf13a33c-306a-11e8-8a29-4d662678931a.png) + + +Scroll down and click on the "Save changes" button: + +![save-changes](https://user-images.githubusercontent.com/194400/37879104-d327af6c-306a-11e8-82d7-48baada8410d.png) - ![ngrok](https://user-images.githubusercontent.com/6057298/34685179-73b6d71c-f49f-11e7-8dab-abfc64c9e938.png) - copy and save the url into your Github App config with ```/event/new``` for endpoint -- View the Project in your Web Browser +### View the App in your Web Browser + +Open http://localhost:4000 in your web browser. You should see something like: + +![gitbu-homepage-localhost](https://user-images.githubusercontent.com/194400/37879114-1f9ca438-306b-11e8-85a9-dec480ef7577.png) + +### Try The App! + + +To test your github-backup server try installing your app +onto one of your repos. + +Visit your App's URL on GitHub e.g: https://github.com/apps/gitbu -Open http://localhost:4000 in your web browser. -To test your github-backup server try installing your app onto one of your repos. You should see the payload of the request in the tab you have open in your terminal for the phoenix server: +You should see the payload of the request in the tab you +have open in your terminal for the phoenix server: ![image](https://user-images.githubusercontent.com/16775804/36433464-77912686-1654-11e8-8a54-0779992d9e18.png) diff --git a/assets/package.json b/assets/package.json index 06faa83..5ed1c29 100644 --- a/assets/package.json +++ b/assets/package.json @@ -24,5 +24,5 @@ "bugs": { "url": "https://github.com/dwyl/github-backup/issues" }, - "homepage": "https://github.com/dwyl/github-backup", + "homepage": "https://github.com/dwyl/github-backup" } From 16519551e4c15b253111660ffa3b8414bb44a967 Mon Sep 17 00:00:00 2001 From: nelsonic Date: Mon, 26 Mar 2018 10:58:21 +0100 Subject: [PATCH 19/23] simplify instructions for adding private key to .env file see https://github.com/dwyl/learn-environment-variables/issues/17#issuecomment-376108747 --- README.md | 18 ++++++++++++++---- keytoenvar.sh | 4 ---- 2 files changed, 14 insertions(+), 8 deletions(-) delete mode 100755 keytoenvar.sh diff --git a/README.md b/README.md index d3dcef0..f3bcd0e 100644 --- a/README.md +++ b/README.md @@ -260,15 +260,14 @@ it's just for demonstration purposes.
but this is what your RSA private key will **look** like, a block of random characters ..._ - The downloaded file contains the private key. Save this file in the root of the `github-backup` repo on your `localhost`. For _example_ in _our_ case the GitHub app is called "**gitbu**",
so our Private Key file starts with the app name -and the date the key was generated: -`gitbu.2018-03-23.private-key.pem` +and the _date_ the key was generated.
+e.g: `gitbu.2018-03-23.private-key.pem` > _**Don't worry**, all `.pem` (private key) files @@ -276,8 +275,15 @@ are ignored in the `.gitignore` file so your file will stay private._ Once you have copied the file into your project root, run the following command: ```sh -source keytoenvar.sh PRIVATE_KEY ./gitbu.2018-03-23.private-key.pem +echo "export PRIVATE_KEY='`cat ./gitbu.2018-03-23.private-key.pem`'" >> .env ``` +Replace the `gitbu.2018-03-23.private-key.pem` part in the command +with the name of _your_ private key file +e.g: `my-github-backup-app.2018-04-01.private-key.pem` + +That will create an entry in the `.env` file for your `PRIVATE_KEY` +environment variable. + #### 7. Copy the App Name and App `ID` from GitHub App Settings Page @@ -425,9 +431,13 @@ Open http://localhost:4000 in your web browser. You should see something like: To test your github-backup server try installing your app onto one of your repos. +#### 1. Visit Your App's GitHub Page + Visit your App's URL on GitHub e.g: https://github.com/apps/gitbu +Install the app on one of your Repositories e.g: + You should see the payload of the request in the tab you have open in your terminal for the phoenix server: diff --git a/keytoenvar.sh b/keytoenvar.sh deleted file mode 100755 index 30482de..0000000 --- a/keytoenvar.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/usr/bin/env bash -file=$2 -name=$1 -export $name="$(awk 'BEGIN{}{out=out$0"\n"}END{print out}' $file| sed 's/\n$//')" From c11173fac276b26bc867ac2bcce64ef92295ed68 Mon Sep 17 00:00:00 2001 From: nelsonic Date: Mon, 26 Mar 2018 11:00:15 +0100 Subject: [PATCH 20/23] update instructions in .env-sample for PRIVATE_KEY --- .env_sample | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.env_sample b/.env_sample index 446577d..6c44c15 100644 --- a/.env_sample +++ b/.env_sample @@ -1,9 +1,6 @@ # APP_HOST is the host of your app, if you deploy the app change to your domain. export APP_HOST=localhost -# PRIVATE_KEY should be generated in your GitHub App settings -# see export PRIVATE_KEY=YOUR_PRIVATE_KEY - # GITHUB_APP_ID is found in settings of your GitHub app under General > About export GITHUB_APP_ID=YOUR_GITHUB_APP_ID @@ -17,3 +14,6 @@ export SECRET_KEY_BASE=run:mix_phx.gen.secret-in-terminal-and-paste-result-here export AWS_ACCESS_KEY_ID=aws-access-key export AWS_SECRET_ACCESS_KEY=super-secret-key export S3_BUCKET_NAME=your-bucket-name + +# PRIVATE_KEY should be generated in your GitHub App settings +# please see instructions in README for how to add it to .env file From 7b58a333a3e4c06a22c3627f33b5c8e4d2aa7ed1 Mon Sep 17 00:00:00 2001 From: nelsonic Date: Mon, 26 Mar 2018 11:22:35 +0100 Subject: [PATCH 21/23] adds screenshots for enabling the app on GitHub for #35 --- README.md | 36 ++++++++++++++++++++++++++++++++++-- 1 file changed, 34 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index f3bcd0e..bb81fec 100644 --- a/README.md +++ b/README.md @@ -435,11 +435,43 @@ onto one of your repos. Visit your App's URL on GitHub e.g: https://github.com/apps/gitbu +![gitbu-homepage-install](https://user-images.githubusercontent.com/194400/37900407-0fa73e64-30e6-11e8-8e40-a4061e315f23.png) -Install the app on one of your Repositories e.g: +Click on the `Install` button +to install the App +for your organisation / repository. + +#### 2. Install the App + +Select the Organisation you wish to install the app for: +![gitbu-install-select-org](https://user-images.githubusercontent.com/194400/37900543-68f3f0c0-30e6-11e8-9b58-f748acebb6ff.png) + +Select the repository you want to install the App for e.g: + +![gitbu-select-respository](https://user-images.githubusercontent.com/194400/37900609-a3b6b742-30e6-11e8-80ed-db34555248a6.png) +then click `Install`. + +You should now see a settings page confirming that the app is installed: + +![gitbu-confirmed-installed](https://user-images.githubusercontent.com/194400/37900752-096ce69c-30e7-11e8-84f2-7efa5d645fff.png) + +#### 3. Create/Update or Close an Issue in the Repo + +In the repo you added the App to, create/update or close an issue. +e.g: + +![gitbu-hits-elixir-issue](https://user-images.githubusercontent.com/194400/37900908-8126d206-30e7-11e8-8dda-fe293409f752.png) + +Since this issue has/had already been resolved by a Pull Request from Sam, +I closed it with a comment: + +![gitbu-issue-closed](https://user-images.githubusercontent.com/194400/37900974-ba20ae88-30e7-11e8-82d1-be88e82cc9d3.png) + +When you perform any Issue action (create, comment, update or close), +you will see a corresponding event in the terminal. You should see the payload of the request in the tab you -have open in your terminal for the phoenix server: +have open in your terminal for the phoenix server, e.g: ![image](https://user-images.githubusercontent.com/16775804/36433464-77912686-1654-11e8-8a54-0779992d9e18.png) From 0c94ae747512aa59633c23b8e9a643262861ab31 Mon Sep 17 00:00:00 2001 From: nelsonic Date: Mon, 26 Mar 2018 12:23:49 +0100 Subject: [PATCH 22/23] adds screenshots of issue data in local PostgreSQL to confirm app working for #35 --- README.md | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index bb81fec..002c117 100644 --- a/README.md +++ b/README.md @@ -467,14 +467,38 @@ I closed it with a comment: ![gitbu-issue-closed](https://user-images.githubusercontent.com/194400/37900974-ba20ae88-30e7-11e8-82d1-be88e82cc9d3.png) +#### 4. Observe the Data in Phoenix App Terminal Window + When you perform any Issue action (create, comment, update or close), you will see a corresponding event in the terminal. You should see the payload of the request in the tab you have open in your terminal for the phoenix server, e.g: -![image](https://user-images.githubusercontent.com/16775804/36433464-77912686-1654-11e8-8a54-0779992d9e18.png) +![gitbu-phoenix-event-terminal-log](https://user-images.githubusercontent.com/194400/37902754-597a2b44-30ed-11e8-905c-48591c2c250b.png) + + +#### 5. Confirm the Data in PostgreSQL Database + +You can _confirm_ that the data was saved to PostgreSQL +by checking your DB.
+Open `pgAdmin` and _navigate_ to the database table +in `app_dev > tables > issues` + +![gitbu-confirm-data-in-postgresql-db](https://user-images.githubusercontent.com/194400/37903055-7a4d04f8-30ee-11e8-8fcd-60304b8d6759.png) + +To _run_ the query, ***right-click*** (**⌘ + click**) on the table +(_in this case `issues`_) and then click "**View/Edit Data**" +followed by "**All Rows**": + +![gitbu-all-rows](https://user-images.githubusercontent.com/194400/37903327-5ed061e2-30ef-11e8-8007-20fabba12f65.png) + +You can _also_ confirm that the _status_ of the issue +has been update to "closed": + +![gitbu-issue_status-closed](https://user-images.githubusercontent.com/194400/37903517-2283910e-30f0-11e8-925d-5dfefff97e42.png) +That's it, your `github-backup` instance is working! ### Deployment From 1046356584815cf49fa3338d17c1c815883bdea2 Mon Sep 17 00:00:00 2001 From: nelsonic Date: Tue, 27 Mar 2018 11:52:13 +0100 Subject: [PATCH 23/23] fix typos noted by @Cleop in https://github.com/dwyl/github-backup/pull/64#pullrequestreview-107227325 (Thanks!) --- .env_sample | 2 +- README.md | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.env_sample b/.env_sample index 6c44c15..88895c2 100644 --- a/.env_sample +++ b/.env_sample @@ -11,7 +11,7 @@ export GITHUB_APP_NAME=NAME_OF_THE_GITHUB_APP export SECRET_KEY_BASE=run:mix_phx.gen.secret-in-terminal-and-paste-result-here # AWS: -export AWS_ACCESS_KEY_ID=aws-access-key +export AWS_ACCESS_KEY_ID=aws-access-key-id export AWS_SECRET_ACCESS_KEY=super-secret-key export S3_BUCKET_NAME=your-bucket-name diff --git a/README.md b/README.md index 002c117..72bdb24 100644 --- a/README.md +++ b/README.md @@ -148,7 +148,7 @@ as the `.env` file is "ignored" in the_ The role of the Github application is to send **notifications** when **events** occur in your repositories.
-For example you can get be notified +For example you can be notified when **new issues** or **pull requests** are ***opened***. > _**Note**: we have **simplified** @@ -329,7 +329,7 @@ You will _also_ need to define the #### "Source" the `.env` File Now that you have defined all the necessary -environment variables in your `.env` file,
it's time to make them _available_ to the App.
Execute the command following command in your terminal: