exclude stocks that have no info in zacks db

cmd/balance_sheet.go

@@ -16,6 +16,8 @@ package cmd
 
 import (
 	"context"
+	"math/rand"
+	"os"
 	"time"
 
 	"github.com/jackc/pgx/v4"
@@ -25,6 +27,11 @@ import (
 	"github.com/spf13/viper"
 )
 
+var (
+	lookback  int
+	maxAssets int
+)
+
 var balanceSheetCmd = &cobra.Command{
 	Use:   "balance-sheet <tickers> ...",
 	Args:  cobra.MinimumNArgs(0),
@@ -39,18 +46,65 @@ var balanceSheetCmd = &cobra.Command{
 		defer conn.Close(ctx)
 
 		if len(args) == 0 {
-			if rows, err := conn.Query(ctx, "SELECT distinct ticker FROM fundamentals WHERE event_date > $1 AND working_capital = 'NaN'::float8", time.Now().Add(-365*24*time.Hour)); err != nil {
+			// get exclusions
+			exclusion := make(map[string]bool)
+			if rows, err := conn.Query(ctx, "SELECT distinct composite_figi FROM zacks_balance_sheet_exclusions"); err != nil {
+				log.Fatal().Err(err).Msg("error querying database for tickers without working_capital")
+			} else {
+				var figi string
+
+				for rows.Next() {
+					if err := rows.Scan(&figi); err != nil {
+						log.Fatal().Err(err).Msg("unable to scan query exclusion into figi string")
+					}
+
+					exclusion[figi] = true
+				}
+			}
+
+			// get tickers
+			if rows, err := conn.Query(ctx, "SELECT distinct ticker, composite_figi FROM fundamentals WHERE event_date > $1 AND working_capital = 'NaN'::float8 AND dim='As-Reported-Quarterly'", time.Now().Add(-90*24*time.Hour)); err != nil {
 				log.Fatal().Err(err).Msg("error querying database for tickers without working_capital")
 			} else {
-				var ticker string
-				if err := rows.Scan(&ticker); err != nil {
-					log.Fatal().Err(err).Msg("unable to scan query value into ticker string")
+				var (
+					ticker string
+					figi   string
+				)
+
+				cnt := 0
+				for rows.Next() {
+					cnt += 1
+					if err := rows.Scan(&ticker, &figi); err != nil {
+						log.Fatal().Err(err).Msg("unable to scan query value into ticker string")
+					}
+
+					if _, ok := exclusion[figi]; !ok {
+						args = append(args, ticker)
+					}
 				}
 
-				args = append(args, ticker)
+				log.Info().Int("Count", cnt).Int("LenArgs", len(args)).Msg("found assets with missing working capital in database")
+			}
+
+			// shuffle the list
+			for i := range args {
+				j := rand.Intn(i + 1)
+				args[i], args[j] = args[j], args[i]
+			}
+
+			log.Info().Int("Count", len(args)).Int("lookback", lookback).Msg("found records missing working_capital")
+
+			// limit run to maxAssets items
+			if len(args) > maxAssets {
+				args = args[:maxAssets]
 			}
 		}
 
+		if len(args) == 0 {
+			log.Error().Msg("No assets to lookup")
+			os.Exit(0)
+		}
+
 		if balanceSheets, err := zacks.BalanceSheet(args); err == nil {
 			log.Info().Int("Count", len(balanceSheets)).Msg("saving balance sheets to database")
 			balanceSheets.SaveToDB(ctx, conn)
@@ -64,5 +118,8 @@ var balanceSheetCmd = &cobra.Command{
 }
 
 func init() {
+	balanceSheetCmd.LocalFlags().IntVar(&lookback, "lookback", 30, "Number of days to lookback")
+	balanceSheetCmd.LocalFlags().IntVar(&maxAssets, "max-assets", 25, "Maximum number of discovered assets to include")
+
 	rootCmd.AddCommand(balanceSheetCmd)
 }

zacks/balance_sheet.go

@@ -52,10 +52,13 @@ func BalanceSheet(tickers []string) (BalanceSheetList, error) {
 			BarEnd:        "]",
 		}))
 
+	completed := 0
 	for _, ticker := range tickers {
 		bar.Describe(ticker)
 
-		if _, err := page.Goto(fmt.Sprintf("https://www.zacks.com/stock/quote/%s/balance-sheet?icid=quote-stock_overview-quote_nav_tracking-zcom-left_subnav_quote_navbar-balance_sheet", ticker), playwright.PageGotoOptions{
+		zacksTicker := strings.ReplaceAll(ticker, "/", ".")
+
+		if _, err := page.Goto(fmt.Sprintf("https://www.zacks.com/stock/quote/%s/balance-sheet?icid=quote-stock_overview-quote_nav_tracking-zcom-left_subnav_quote_navbar-balance_sheet", zacksTicker), playwright.PageGotoOptions{
 			WaitUntil: playwright.WaitUntilStateNetworkidle,
 			Timeout:   playwright.Float(20000),
 		}); err != nil {
@@ -72,7 +75,11 @@ func BalanceSheet(tickers []string) (BalanceSheetList, error) {
 		// get section header
 		annual := make(map[string]*BalanceSheetRecord, 5)
 		colMap := make(map[int]string, 5)
-		parseHeader("#annual_income_statement", ticker, "As-Reported-Annual", page, annual, colMap)
+		if err := parseHeader("#annual_income_statement", ticker, "As-Reported-Annual", page, annual, colMap); err != nil {
+			// add to database
+			AddExclusion(ticker)
+			continue
+		}
 
 		if len(colMap) > 0 {
 			// current assets
@@ -81,9 +88,12 @@ func BalanceSheet(tickers []string) (BalanceSheetList, error) {
 			parseRow("#annual_income_statement", "Total Current Liabilities", "TotalCurrentLiabilities", page, annual, colMap)
 		}
 
+		allNaN := true
+
 		// add all ARY dimension to return val
 		for _, v := range annual {
 			result = append(result, v)
+			allNaN = (math.IsNaN(v.TotalCurrentAssets) && math.IsNaN(v.TotalCurrentLiabilities)) && allNaN
 		}
 
 		// Quarterly Income Statement
@@ -109,8 +119,22 @@ func BalanceSheet(tickers []string) (BalanceSheetList, error) {
 		// add all ARQ dimension to return val
 		for _, v := range quarterly {
 			result = append(result, v)
+			allNaN = (math.IsNaN(v.TotalCurrentAssets) && math.IsNaN(v.TotalCurrentLiabilities)) && allNaN
 		}
+
 		bar.Add(1)
+		completed += 1
+
+		if allNaN {
+			AddExclusion(ticker)
+		}
+
+		// every 50 tickers restart playwright
+		if completed > 50 {
+			common.StopPlaywright(page, context, browser, pw)
+			page, context, browser, pw = common.StartPlaywright(viper.GetBool("playwright.headless"))
+			completed = 0
+		}
 	}
 
 	common.StopPlaywright(page, context, browser, pw)
@@ -170,6 +194,9 @@ func parseRow(selector string, rowLabel string, fieldName string, page playwrigh
 				log.Error().Err(err).Str("inputVal", val).Str("column", colMap[idx]).Msg("could not convert value to float")
 			} else {
 				floatVal *= 1e6
+				if floatVal < 0 {
+					floatVal = math.NaN()
+				}
 				reflect.ValueOf(table[colMap[idx]]).Elem().FieldByName(fieldName).Set(reflect.ValueOf(floatVal))
 			}
 		}

zacks/database.go

@@ -24,6 +24,34 @@ import (
 	"github.com/spf13/viper"
 )
 
+func AddExclusion(ticker string) {
+	ctx := context.Background()
+
+	conn, err := pgx.Connect(ctx, viper.GetString("database.url"))
+	if err != nil {
+		log.Error().Err(err).Msg("could not connect to database in AddExclusion")
+	}
+	defer conn.Close(ctx)
+
+	// lookup the composite figi
+	rows, err := conn.Query(ctx, "SELECT composite_figi FROM assets WHERE ticker=$1 AND active='t' LIMIT 1", ticker)
+	if err != nil {
+		log.Error().Err(err).Msg("could not query database from composite_figi in AddExclusion")
+	}
+
+	var figi string
+	for rows.Next() {
+		if err := rows.Scan(&figi); err != nil {
+			log.Error().Err(err).Msg("could not scan into figi string")
+		}
+	}
+
+	// save to exclusions table
+	if _, err := conn.Exec(ctx, `INSERT INTO zacks_balance_sheet_exclusions ("ticker", "composite_figi") VALUES ($1, $2)`, ticker, figi); err != nil {
+		log.Error().Err(err).Str("Ticker", ticker).Str("CompositeFIGI", figi).Msg("could not save exclusion to DB")
+	}
+}
+
 func SaveToDB(records []*ZacksRecord) error {
 	conn, err := pgx.Connect(context.Background(), viper.GetString("database.url"))
 	if err != nil {