Merge pull request #19 from lawzava/general-maintenance

🎨 general maintenance

.github/workflows/golang.yml

@@ -9,7 +9,7 @@ jobs:
       - name: Setup Go for use with actions
         uses: actions/setup-go@v2
         with:
-          go-version: 1.16
+          go-version: 1.19
       - uses: actions/checkout@v2
       - uses: actions/cache@v2
         with:

.github/workflows/golangci.yml

@@ -10,4 +10,4 @@ jobs:
       - name: golangci-lint
         uses: golangci/golangci-lint-action@v2
         with:
-          version: v1.41.1
+          version: v1.50.0

.github/workflows/release.yml

@@ -20,10 +20,10 @@ jobs:
         name: Set up Go
         uses: actions/setup-go@v1
         with:
-          go-version: 1.16.x
+          go-version: 1.19.x
       -
         name: Run GoReleaser
-        uses: goreleaser/goreleaser-action@v2
+        uses: goreleaser/goreleaser-action@v3
         with:
           version: latest
           args: release --rm-dist

.golangci.yml

@@ -7,7 +7,7 @@ linters-settings:
       - performance
       - style
   goimports:
-    local-prefixes: github.com/golangci/golangci-lint
+    local-prefixes: github.com/lawzava/scrape
   govet:
     check-shadowing: true
   funlen:
@@ -20,84 +20,7 @@ linters-settings:
     extra-rules: true
 
 linters:
-  disable-all: true
-  enable:
-    - bodyclose
-    - deadcode
-    - depguard
-    - dogsled
-    - dupl
-    - errcheck
-    - funlen
-    - gochecknoglobals
-    - gochecknoinits
-    - gocognit
-    - goconst
-    - gocritic
-    - gocyclo
-    - godot
-    - gofmt
-    - goimports
-    - gomnd
-    - gomodguard
-    - goprintffuncname
-    - gosec
-    - gosimple
-    - govet
-    - ineffassign
-    - lll
-    - misspell
-    - nakedret
-    - nestif
-    - prealloc
-    - rowserrcheck
-    - staticcheck
-    - structcheck
-    - stylecheck
-    - typecheck
-    - unconvert
-    - unparam
-    - unused
-    - varcheck
-    - whitespace
-    - wsl
-    - asciicheck
-    - godox
-    - nolintlint
-    - goerr113
-    - exhaustive
-    - exportloopref
-    - gofumpt
-    - goheader
-    - noctx
-    - sqlclosecheck
-    - nlreturn
-    - errorlint
-    - exhaustivestruct
-    - paralleltest
-    - tparallel
-    - wrapcheck
-    - forbidigo
-    - makezero
-    - predeclared
-    - thelper
-    - ifshort
-    - cyclop
-    - durationcheck
-    - forcetypeassert
-    - gci
-    - gomoddirectives
-    - importas
-    - nilerr
-    - promlinter
-    - revive
-    - tagliatelle
-    - testpackage
-    - wastedassign
-    # - interfacer # Deprecated
-    # - golint # Deprecated
-    # - maligned # Deprecated
-    # - scopelint # Deprecated
+  enable-all: true
 
 issues:
   exclude-rules:

.goreleaser.yml

@@ -8,7 +8,6 @@ builds:
  -
   env:
     - CGO_ENABLED=0
-  main: ./main.go
   goos:
     - linux
     - darwin

cli.go

@@ -0,0 +1,37 @@
+package main
+
+import (
+	"github.com/lawzava/emailscraper"
+)
+
+//nolint:gochecknoglobals // allow global var here
+var (
+	scraperParameters emailscraper.Config
+	url               string
+	output            string
+	outputWithURL     bool
+)
+
+//nolint:gochecknoinits // required by github.com/spf13/cobra
+func init() {
+	rootCmd.PersistentFlags().StringVarP(&url,
+		"website", "w", "https://lawzava.com", "Website to scrape")
+	rootCmd.PersistentFlags().BoolVar(&scraperParameters.Recursively,
+		"recursively", true, "Scrape website recursively")
+	rootCmd.PersistentFlags().IntVarP(&scraperParameters.MaxDepth,
+		"depth", "d", 3, "Max depth to follow when scraping recursively") //nolint:gomnd // allow default max depth
+	rootCmd.PersistentFlags().BoolVar(&scraperParameters.Async,
+		"async", true, "Scrape website pages asynchronously")
+	rootCmd.PersistentFlags().BoolVar(&scraperParameters.Debug,
+		"debug", false, "Print debug logs")
+	rootCmd.PersistentFlags().BoolVar(&scraperParameters.FollowExternalLinks,
+		"follow-external", false, "Follow external 3rd party links within website")
+	rootCmd.PersistentFlags().BoolVar(&scraperParameters.EnableJavascript,
+		"js", false, "Enables EnableJavascript execution await")
+	rootCmd.PersistentFlags().IntVar(&scraperParameters.Timeout,
+		"timeout", 0, "If > 0, specify a timeout (seconds) for js execution await")
+	rootCmd.PersistentFlags().StringVar(&output,
+		"output", outputPlain, "Output type to use (default 'plain', supported: 'csv', 'json')")
+	rootCmd.PersistentFlags().BoolVar(&outputWithURL,
+		"output-with-url", false, "Adds URL to output with each email")
+}

go.mod

@@ -1,12 +1,39 @@
 module github.com/lawzava/scrape
 
-go 1.16
+go 1.19
 
 require (
-	github.com/antchfx/xmlquery v1.3.8 // indirect
-	github.com/chromedp/cdproto v0.0.0-20211019232255-96776d03ee97 // indirect
-	github.com/lawzava/emailscraper v1.1.3
-	github.com/spf13/cobra v1.2.1
-	golang.org/x/net v0.0.0-20211020060615-d418f374d309 // indirect
-	golang.org/x/sys v0.0.0-20211020154033-fcb26fe61c20 // indirect
+	github.com/lawzava/emailscraper v1.2.0
+	github.com/spf13/cobra v1.6.0
+)
+
+require (
+	github.com/PuerkitoBio/goquery v1.8.0 // indirect
+	github.com/andybalholm/cascadia v1.3.1 // indirect
+	github.com/antchfx/htmlquery v1.2.5 // indirect
+	github.com/antchfx/xmlquery v1.3.12 // indirect
+	github.com/antchfx/xpath v1.2.1 // indirect
+	github.com/chromedp/cdproto v0.0.0-20221011223153-490dc4d81f7c // indirect
+	github.com/chromedp/chromedp v0.8.6 // indirect
+	github.com/chromedp/sysutil v1.0.0 // indirect
+	github.com/gobwas/glob v0.2.3 // indirect
+	github.com/gobwas/httphead v0.1.0 // indirect
+	github.com/gobwas/pool v0.2.1 // indirect
+	github.com/gobwas/ws v1.1.0 // indirect
+	github.com/gocolly/colly/v2 v2.1.0 // indirect
+	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
+	github.com/golang/protobuf v1.5.2 // indirect
+	github.com/inconshreveable/mousetrap v1.0.1 // indirect
+	github.com/josharian/intern v1.0.0 // indirect
+	github.com/kennygrant/sanitize v1.2.4 // indirect
+	github.com/lawzava/go-tld v1.0.1 // indirect
+	github.com/mailru/easyjson v0.7.7 // indirect
+	github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca // indirect
+	github.com/spf13/pflag v1.0.5 // indirect
+	github.com/temoto/robotstxt v1.1.2 // indirect
+	golang.org/x/net v0.0.0-20221014081412-f15817d10f9b // indirect
+	golang.org/x/sys v0.0.0-20221013171732-95e765b1cc43 // indirect
+	golang.org/x/text v0.3.8 // indirect
+	google.golang.org/appengine v1.6.7 // indirect
+	google.golang.org/protobuf v1.28.1 // indirect
 )