feat(cli): Add an argument to limit the number of repositories analyzed

.golangci.yml

@@ -44,6 +44,7 @@ linters:
     - funlen
     - gocyclo
     - varnamelen
+    - ifshort # bug: https://github.com/esimonov/ifshort/issues/23
   disable-all: false
   presets:
     - bugs

cmd/src-fingerprint/main.go

@@ -21,13 +21,17 @@ var date = "unknown"
 
 const MaxPipelineEvents = 100
 
-func runExtract(pipeline *srcfingerprint.Pipeline, user string, after string) chan srcfingerprint.PipelineEvent {
+func runExtract(
+	pipeline *srcfingerprint.Pipeline,
+	user string,
+	after string,
+	limit int) chan srcfingerprint.PipelineEvent {
 	// buffer it a bit so it won't block if this is going too fast
 	ch := make(chan srcfingerprint.PipelineEvent, MaxPipelineEvents)
 
 	go func(eventChannel chan srcfingerprint.PipelineEvent) {
 		defer close(eventChannel)
-		pipeline.ExtractRepositories(user, after, eventChannel)
+		pipeline.ExtractRepositories(user, after, eventChannel, limit)
 	}(ch)
 
 	return ch
@@ -71,6 +75,7 @@ type authorInfo struct {
 }
 
 const DefaultClonerN = 8
+const DefaultLimit = 100
 
 func main() {
 	cli.VersionFlag = &cli.BoolFlag{
@@ -152,6 +157,11 @@ func main() {
 				Name:  "provider-url",
 				Usage: "base URL of the Git provider API. If not set, defaults URL are used.",
 			},
+			&cli.IntFlag{
+				Name:  "limit",
+				Value: DefaultLimit,
+				Usage: "maximum number of repositories to analyze (0 for unlimited).",
+			},
 		},
 		Action: mainAction,
 	}
@@ -165,7 +175,7 @@ func mainAction(c *cli.Context) error {
 	if c.Bool("verbose") {
 		log.SetLevel(log.InfoLevel)
 	} else {
-		log.SetLevel(log.ErrorLevel)
+		log.SetLevel(log.WarnLevel)
 	}
 
 	output := os.Stdout
@@ -226,7 +236,7 @@ func mainAction(c *cli.Context) error {
 
 	ticker := time.Tick(1 * time.Second)
 
-	eventChannel := runExtract(&pipeline, c.String("object"), c.String("after"))
+	eventChannel := runExtract(&pipeline, c.String("object"), c.String("after"), c.Int("limit"))
 
 	// runtime stats
 	var (

extractor.go

@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"os"
 	"os/exec"
+	"regexp"
 
 	log "github.com/sirupsen/logrus"
 )
@@ -72,9 +73,13 @@ func (fe *FastExtractor) Run(path string, after string) chan *GitFile {
 
 			var gitFile GitFile
 
-			err := json.Unmarshal(line, &gitFile)
+			// Replace backslashes by escaped backslashes
+			re := regexp.MustCompile(`\\\\(.)`)
+			cleanedLine := re.ReplaceAll(line, []byte("\\\\$1"))
+
+			err := json.Unmarshal(cleanedLine, &gitFile)
 			if err != nil {
-				log.Warnln(err)
+				log.Warnln("Error while parsing", string(line), err)
 			}
 
 			fe.ChanGitFiles <- &gitFile

pipeline.go

@@ -69,7 +69,8 @@ func (p *Pipeline) gather(
 	wg *sync.WaitGroup,
 	eventChan chan<- PipelineEvent,
 	user string,
-	output chan<- provider.GitRepository) {
+	output chan<- provider.GitRepository,
+	limit int) {
 	defer wg.Done()
 	defer close(output)
 
@@ -82,8 +83,21 @@ func (p *Pipeline) gather(
 
 	p.publishEvent(eventChan, RepositoryListPipelineEvent{repositories})
 
-	for _, repository := range repositories {
-		output <- repository
+	collected := 0
+	ignored := 0
+
+	for index, repository := range repositories {
+		if limit > 0 && index >= limit {
+			ignored++
+		} else {
+			collected++
+			output <- repository
+		}
+	}
+
+	if ignored > 0 {
+		log.Warnln("Limit reached for number of repositories")
+		log.Warnf("Collected %d repos, ignored %d repos.", collected, ignored)
 	}
 
 	log.Infoln("Done gathering repositories")
@@ -119,7 +133,7 @@ const (
 )
 
 // ExtractRepositories extract repositories and analyze it for a given user and provider.
-func (p *Pipeline) ExtractRepositories(user string, after string, eventChan chan<- PipelineEvent) {
+func (p *Pipeline) ExtractRepositories(user string, after string, eventChan chan<- PipelineEvent, limit int) {
 	log.Infof("Extracting user %v\n", user)
 
 	repositoryChannel := make(chan provider.GitRepository)
@@ -133,7 +147,7 @@ func (p *Pipeline) ExtractRepositories(user string, after string, eventChan chan
 
 	wg.Add(1)
 
-	go p.gather(&wg, eventChan, user, repositoryChannel)
+	go p.gather(&wg, eventChan, user, repositoryChannel, limit)
 
 	for i := 0; i < extractionWorkersCount; i++ {
 		wg.Add(1)

pipeline_test.go

@@ -67,7 +67,33 @@ func (suite *PipelineTestSuite) TestGather() {
 	providerMock.On("Gather", "user").Return([]provider.GitRepository{createGitRepository("1")}, nil)
 
 	wg.Add(1)
-	go pipeline.gather(wg, nil, "user", outputChan)
+	go pipeline.gather(wg, nil, "user", outputChan, 0)
+
+	repositories := make([]provider.GitRepository, 0, 2)
+	for output := range outputChan {
+		repositories = append(repositories, output)
+	}
+	wg.Wait()
+
+	providerMock.AssertExpectations(suite.T())
+	assert.Equal(suite.T(), []provider.GitRepository{gitRepositoryMock{name: "1"}}, repositories)
+}
+
+func (suite *PipelineTestSuite) TestGatherWithLimit() {
+	outputChan := make(chan provider.GitRepository)
+	wg := &sync.WaitGroup{}
+	providerMock := &ProviderMock{}
+	pipeline := Pipeline{
+		Provider: providerMock,
+	}
+
+	providerMock.On("Gather", "user").Return(
+		[]provider.GitRepository{createGitRepository("1"), createGitRepository("2")},
+		nil,
+	)
+
+	wg.Add(1)
+	go pipeline.gather(wg, nil, "user", outputChan, 1)
 
 	repositories := make([]provider.GitRepository, 0, 2)
 	for output := range outputChan {
@@ -136,7 +162,7 @@ func (suite *PipelineTestSuite) TestExtractRepositories() {
 	go func() {
 		defer close(eventChan)
 
-		pipeline.ExtractRepositories("user", "", eventChan)
+		pipeline.ExtractRepositories("user", "", eventChan, 0)
 	}()
 
 	events := make([]PipelineEvent, 0)