From 6b396f5f27f6680aa81b9c826f65cbfe3e3d80b2 Mon Sep 17 00:00:00 2001 From: Dennis Trautwein Date: Wed, 4 Oct 2023 09:47:54 +0200 Subject: [PATCH] Add support for a helia container --- cmd_run.go | 57 +++++++++++++------ db_client.go | 5 +- db_dummy.go | 7 ++- docker-compose.yml | 7 +-- .../000004_add_providers_table.down.sql | 1 + ...0006_rename_kubo_measurement_type.down.sql | 1 + ...000006_rename_kubo_measurement_type.up.sql | 1 + ...dd_ipfs_impl_column_to_runs_table.down.sql | 6 ++ ..._add_ipfs_impl_column_to_runs_table.up.sql | 11 ++++ models/boil_types.go | 4 +- models/runs.go | 11 +++- probe.go | 20 +++---- providers.go | 6 +- 13 files changed, 93 insertions(+), 44 deletions(-) create mode 100644 migrations/000006_rename_kubo_measurement_type.down.sql create mode 100644 migrations/000006_rename_kubo_measurement_type.up.sql create mode 100644 migrations/000007_add_ipfs_impl_column_to_runs_table.down.sql create mode 100644 migrations/000007_add_ipfs_impl_column_to_runs_table.up.sql diff --git a/cmd_run.go b/cmd_run.go index 9769318..15a7da0 100644 --- a/cmd_run.go +++ b/cmd_run.go @@ -43,6 +43,12 @@ var RunCommand = &cli.Command{ EnvVars: []string{"TIROS_RUN_TIMES"}, Value: 3, }, + &cli.BoolFlag{ + Name: "lookup-providers", + Usage: "Whether to lookup website providers", + EnvVars: []string{"TIROS_RUN_LOOKUP_PROVIDERS"}, + Value: true, + }, &cli.BoolFlag{ Name: "dry-run", Usage: "Whether to skip DB interactions", @@ -79,21 +85,21 @@ var RunCommand = &cli.Command{ EnvVars: []string{"TIROS_RUN_DATABASE_SSL_MODE"}, }, &cli.StringFlag{ - Name: "kubo-host", - Usage: "port to reach the Kubo Gateway", - EnvVars: []string{"TIROS_RUN_KUBO_HOST"}, + Name: "ipfs-host", + Usage: "host at which to reach the IPFS Gateway", + EnvVars: []string{"TIROS_RUN_IPFS_HOST", "TIROS_RUN_KUBO_HOST" /* <- legacy */}, Value: "localhost", }, &cli.IntFlag{ - Name: "kubo-api-port", - Usage: "port to reach the Kubo API", - EnvVars: []string{"TIROS_RUN_KUBO_API_PORT"}, + Name: "ipfs-api-port", + Usage: "port to reach a Kubo-compatible RPC API", + EnvVars: []string{"TIROS_RUN_IPFS_API_PORT", "TIROS_RUN_KUBO_API_PORT" /* <- legacy */}, Value: 5001, }, &cli.IntFlag{ - Name: "kubo-gateway-port", - Usage: "port to reach the Kubo Gateway", - EnvVars: []string{"TIROS_RUN_KUBO_GATEWAY_PORT"}, + Name: "ipfs-gateway-port", + Usage: "port to reach the IPFS Gateway", + EnvVars: []string{"TIROS_RUN_IPFS_GATEWAY_PORT", "TIROS_RUN_KUBO_GATEWAY_PORT" /* <- legacy */}, Value: 8080, }, &cli.IntFlag{ @@ -119,13 +125,19 @@ var RunCommand = &cli.Command{ Usage: "Path to the Udger DB", EnvVars: []string{"TIROS_UDGER_DB_PATH"}, }, + &cli.StringFlag{ + Name: "ipfs-implementation", + Usage: "Which implementation are we testing (KUBO, HELIA)", + EnvVars: []string{"TIROS_IPFS_IMPLEMENTATION"}, + Value: "KUBO", + }, }, Action: RunAction, } type tiros struct { dbClient IDBClient - kubo *shell.Shell + ipfs *shell.Shell dbRun *models.Run mmClient *maxmind.Client uClient *udger.Client @@ -145,8 +157,11 @@ func RunAction(c *cli.Context) error { } } - // Initialize kubo client - kubo := shell.NewShell(fmt.Sprintf("/ip4/127.0.0.1/tcp/%d", c.Int("kubo-api-port"))) + // Initialize ipfs client + var ipfsClient *shell.Shell + if c.Int("ipfs-api-port") != 0 { + ipfsClient = shell.NewShell(fmt.Sprintf("/ip4/127.0.0.1/tcp/%d", c.Int("ipfs-api-port"))) + } // Initialize maxmind client mmClient, err := maxmind.NewClient() @@ -163,7 +178,7 @@ func RunAction(c *cli.Context) error { // configure tiros struct t := tiros{ dbClient: dbClient, - kubo: kubo, + ipfs: ipfsClient, mmClient: mmClient, uClient: uClient, } @@ -182,7 +197,7 @@ func RunAction(c *cli.Context) error { }() // shuffle websites, so that we have a different order in which we request the websites. - // If we didn't do this a single website would always be requested with a comparatively "cold" kubo node. + // If we didn't do this a single website would always be requested with a comparatively "cold" ipfs node. websites := c.StringSlice("websites") rand.Seed(time.Now().UnixNano()) rand.Shuffle(len(websites), func(i, j int) { @@ -199,7 +214,12 @@ func RunAction(c *cli.Context) error { probeResults := make(chan *probeResult) go t.measureWebsites(c, websites, probeResults) - go t.findAllProviders(c, websites, providerResults) + + if c.Bool("lookup-providers") { + go t.findAllProviders(c, websites, providerResults) + } else { + close(providerResults) + } for { select { @@ -242,12 +262,13 @@ func RunAction(c *cli.Context) error { } func (t *tiros) InitRun(c *cli.Context) (*models.Run, error) { - version, sha, err := t.kubo.Version() + version, sha, err := t.ipfs.Version() if err != nil { - return nil, fmt.Errorf("kubo api offline: %w", err) + return nil, fmt.Errorf("ipfs api offline: %w", err) } - dbRun, err := t.dbClient.InsertRun(c, fmt.Sprintf("%s-%s", version, sha)) + ipfsImpl := c.String("ipfs-implementation") + dbRun, err := t.dbClient.InsertRun(c, ipfsImpl, fmt.Sprintf("%s-%s", version, sha)) if err != nil { return nil, fmt.Errorf("insert run: %w", err) } diff --git a/db_client.go b/db_client.go index a5d6ae4..12256a9 100644 --- a/db_client.go +++ b/db_client.go @@ -28,7 +28,7 @@ import ( var migrations embed.FS type IDBClient interface { - InsertRun(c *cli.Context, version string) (*models.Run, error) + InsertRun(c *cli.Context, ipfsImpl string, version string) (*models.Run, error) SaveMeasurement(c *cli.Context, dbRun *models.Run, pr *probeResult) (*models.Measurement, error) SaveProvider(c *cli.Context, dbRun *models.Run, provider *provider) (*models.Provider, error) SealRun(ctx context.Context, dbRun *models.Run) (*models.Run, error) @@ -135,7 +135,7 @@ func (db *DBClient) SealRun(ctx context.Context, dbRun *models.Run) (*models.Run return dbRun, err } -func (db *DBClient) InsertRun(c *cli.Context, version string) (*models.Run, error) { +func (db *DBClient) InsertRun(c *cli.Context, ipfsImpl string, version string) (*models.Run, error) { log.Infoln("Inserting Run...") websites := make([]string, len(c.StringSlice("websites"))) @@ -148,6 +148,7 @@ func (db *DBClient) InsertRun(c *cli.Context, version string) (*models.Run, erro Region: c.String("region"), Websites: websites, Version: version, + IpfsImpl: ipfsImpl, Times: int16(c.Int("times")), CPU: c.Int("cpu"), Memory: c.Int("memory"), diff --git a/db_dummy.go b/db_dummy.go index 421e81c..757c4d7 100644 --- a/db_dummy.go +++ b/db_dummy.go @@ -24,10 +24,11 @@ func (D DBDummyClient) SealRun(ctx context.Context, dbRun *models.Run) (*models. return nil, nil } -func (D DBDummyClient) InsertRun(c *cli.Context, version string) (*models.Run, error) { +func (D DBDummyClient) InsertRun(c *cli.Context, ipfsImpl string, version string) (*models.Run, error) { return &models.Run{ - ID: 2, - Region: "dummy", + ID: 2, + Region: "dummy", + IpfsImpl: ipfsImpl, }, nil } diff --git a/docker-compose.yml b/docker-compose.yml index 2216e14..abaf563 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,16 +2,15 @@ version: "3.9" name: tiros services: ipfs: - image: ipfs/kubo:v0.19.0 + image: helia:latest restart: unless-stopped volumes: - ipfs_path:/data/ipfs - ipfs_fuse:/ipfs - ipns_fuse:/ipns + environment: + DEBUG: helia-server ports: - - "4001:4001/tcp" - - "4001:4001/udp" - - "0.0.0.0:5001:5001" - "0.0.0.0:8080:8080" chrome: image: browserless/chrome:latest diff --git a/migrations/000004_add_providers_table.down.sql b/migrations/000004_add_providers_table.down.sql index e69de29..eb08829 100644 --- a/migrations/000004_add_providers_table.down.sql +++ b/migrations/000004_add_providers_table.down.sql @@ -0,0 +1 @@ +DROP TABLE providers; \ No newline at end of file diff --git a/migrations/000006_rename_kubo_measurement_type.down.sql b/migrations/000006_rename_kubo_measurement_type.down.sql new file mode 100644 index 0000000..1b64662 --- /dev/null +++ b/migrations/000006_rename_kubo_measurement_type.down.sql @@ -0,0 +1 @@ +ALTER TYPE measurement_type RENAME VALUE 'IPFS' TO 'KUBO'; \ No newline at end of file diff --git a/migrations/000006_rename_kubo_measurement_type.up.sql b/migrations/000006_rename_kubo_measurement_type.up.sql new file mode 100644 index 0000000..1f8ff28 --- /dev/null +++ b/migrations/000006_rename_kubo_measurement_type.up.sql @@ -0,0 +1 @@ +ALTER TYPE measurement_type RENAME VALUE 'KUBO' TO 'IPFS'; \ No newline at end of file diff --git a/migrations/000007_add_ipfs_impl_column_to_runs_table.down.sql b/migrations/000007_add_ipfs_impl_column_to_runs_table.down.sql new file mode 100644 index 0000000..51472e4 --- /dev/null +++ b/migrations/000007_add_ipfs_impl_column_to_runs_table.down.sql @@ -0,0 +1,6 @@ +BEGIN; + +ALTER TABLE runs + DROP COLUMN ipfs_impl; + +COMMIT; \ No newline at end of file diff --git a/migrations/000007_add_ipfs_impl_column_to_runs_table.up.sql b/migrations/000007_add_ipfs_impl_column_to_runs_table.up.sql new file mode 100644 index 0000000..4d8c5a0 --- /dev/null +++ b/migrations/000007_add_ipfs_impl_column_to_runs_table.up.sql @@ -0,0 +1,11 @@ +BEGIN; + +ALTER TABLE runs + ADD COLUMN ipfs_impl TEXT; + +UPDATE runs SET ipfs_impl = 'KUBO'; + +ALTER TABLE runs + ALTER COLUMN ipfs_impl SET NOT NULL; + +COMMIT; \ No newline at end of file diff --git a/models/boil_types.go b/models/boil_types.go index dc71d2a..1f531ca 100644 --- a/models/boil_types.go +++ b/models/boil_types.go @@ -54,13 +54,13 @@ func makeCacheKey(cols boil.Columns, nzDefaults []string) string { // Enum values for MeasurementType const ( MeasurementTypeHTTP string = "HTTP" - MeasurementTypeKUBO string = "KUBO" + MeasurementTypeIPFS string = "IPFS" ) func AllMeasurementType() []string { return []string{ MeasurementTypeHTTP, - MeasurementTypeKUBO, + MeasurementTypeIPFS, } } diff --git a/models/runs.go b/models/runs.go index 6986015..ce9020c 100644 --- a/models/runs.go +++ b/models/runs.go @@ -35,6 +35,7 @@ type Run struct { UpdatedAt time.Time `boil:"updated_at" json:"updated_at" toml:"updated_at" yaml:"updated_at"` CreatedAt time.Time `boil:"created_at" json:"created_at" toml:"created_at" yaml:"created_at"` FinishedAt null.Time `boil:"finished_at" json:"finished_at,omitempty" toml:"finished_at" yaml:"finished_at,omitempty"` + IpfsImpl string `boil:"ipfs_impl" json:"ipfs_impl" toml:"ipfs_impl" yaml:"ipfs_impl"` R *runR `boil:"-" json:"-" toml:"-" yaml:"-"` L runL `boil:"-" json:"-" toml:"-" yaml:"-"` @@ -51,6 +52,7 @@ var RunColumns = struct { UpdatedAt string CreatedAt string FinishedAt string + IpfsImpl string }{ ID: "id", Region: "region", @@ -62,6 +64,7 @@ var RunColumns = struct { UpdatedAt: "updated_at", CreatedAt: "created_at", FinishedAt: "finished_at", + IpfsImpl: "ipfs_impl", } var RunTableColumns = struct { @@ -75,6 +78,7 @@ var RunTableColumns = struct { UpdatedAt string CreatedAt string FinishedAt string + IpfsImpl string }{ ID: "runs.id", Region: "runs.region", @@ -86,6 +90,7 @@ var RunTableColumns = struct { UpdatedAt: "runs.updated_at", CreatedAt: "runs.created_at", FinishedAt: "runs.finished_at", + IpfsImpl: "runs.ipfs_impl", } // Generated where @@ -125,6 +130,7 @@ var RunWhere = struct { UpdatedAt whereHelpertime_Time CreatedAt whereHelpertime_Time FinishedAt whereHelpernull_Time + IpfsImpl whereHelperstring }{ ID: whereHelperint{field: "\"runs\".\"id\""}, Region: whereHelperstring{field: "\"runs\".\"region\""}, @@ -136,6 +142,7 @@ var RunWhere = struct { UpdatedAt: whereHelpertime_Time{field: "\"runs\".\"updated_at\""}, CreatedAt: whereHelpertime_Time{field: "\"runs\".\"created_at\""}, FinishedAt: whereHelpernull_Time{field: "\"runs\".\"finished_at\""}, + IpfsImpl: whereHelperstring{field: "\"runs\".\"ipfs_impl\""}, } // RunRels is where relationship names are stored. @@ -176,8 +183,8 @@ func (r *runR) GetProviders() ProviderSlice { type runL struct{} var ( - runAllColumns = []string{"id", "region", "websites", "version", "times", "cpu", "memory", "updated_at", "created_at", "finished_at"} - runColumnsWithoutDefault = []string{"region", "websites", "version", "times", "cpu", "memory", "updated_at", "created_at"} + runAllColumns = []string{"id", "region", "websites", "version", "times", "cpu", "memory", "updated_at", "created_at", "finished_at", "ipfs_impl"} + runColumnsWithoutDefault = []string{"region", "websites", "version", "times", "cpu", "memory", "updated_at", "created_at", "ipfs_impl"} runColumnsWithDefault = []string{"id", "finished_at"} runPrimaryKeyColumns = []string{"id"} runGeneratedColumns = []string{"id"} diff --git a/probe.go b/probe.go index dedac12..da37d04 100644 --- a/probe.go +++ b/probe.go @@ -38,11 +38,11 @@ func (t *tiros) measureWebsites(c *cli.Context, websites []string, results chan< sleepDur := time.Duration(settle) * time.Second - log.Infof("Letting Kubo settle for %s\n", sleepDur) + log.Infof("Letting the IPFS implementation settle for %s\n", sleepDur) time.Sleep(sleepDur) for i := 0; i < c.Int("times"); i++ { - for _, mType := range []string{models.MeasurementTypeKUBO, models.MeasurementTypeHTTP} { + for _, mType := range []string{models.MeasurementTypeIPFS, models.MeasurementTypeHTTP} { for _, website := range websites { pr, err := newProbe(c, website, mType).run() @@ -67,9 +67,9 @@ func (t *tiros) measureWebsites(c *cli.Context, websites []string, results chan< results <- pr - if mType == models.MeasurementTypeKUBO { - if err = t.KuboGC(c.Context); err != nil { - log.WithError(err).Warnln("error running kubo gc") + if mType == models.MeasurementTypeIPFS { + if err = t.GarbageCollect(c.Context); err != nil { + log.WithError(err).Warnln("error running ipfs gc") continue } } @@ -98,7 +98,7 @@ type probeResult struct { url string website string - // measurement type (KUBO or HTTP) + // measurement type (IPFS or HTTP) mType string try int @@ -329,8 +329,8 @@ func (p *probe) close() { func websiteURL(c *cli.Context, website string, mType string) string { switch mType { - case models.MeasurementTypeKUBO: - return fmt.Sprintf("http://%s:%d/ipns/%s", c.String("kubo-host"), c.Int("kubo-gateway-port"), website) + case models.MeasurementTypeIPFS: + return fmt.Sprintf("http://%s:%d/ipns/%s", c.String("ipfs-host"), c.Int("ipfs-gateway-port"), website) case models.MeasurementTypeHTTP: return fmt.Sprintf("https://%s", website) default: @@ -338,8 +338,8 @@ func websiteURL(c *cli.Context, website string, mType string) string { } } -func (t *tiros) KuboGC(ctx context.Context) error { - return t.kubo.Request("repo/gc").Exec(ctx, nil) +func (t *tiros) GarbageCollect(ctx context.Context) error { + return t.ipfs.Request("repo/gc").Exec(ctx, nil) } func p2f(ptr *float64) float64 { diff --git a/providers.go b/providers.go index 2b59680..c4c69cd 100644 --- a/providers.go +++ b/providers.go @@ -56,7 +56,7 @@ func (t *tiros) findProviders(ctx context.Context, website string, results chan< logEntry := log.WithField("website", website) logEntry.Infoln("Finding providers for", website) - nameResp, err := t.kubo.Request("name/resolve"). + nameResp, err := t.ipfs.Request("name/resolve"). Option("arg", website). Option("nocache", "true"). Option("dht-timeout", "30s").Send(ctx) @@ -87,7 +87,7 @@ func (t *tiros) findProviders(ctx context.Context, website string, results chan< return fmt.Errorf("unmarshal name/resolve response: %w", err) } - findResp, err := t.kubo. + findResp, err := t.ipfs. Request("routing/findprovs"). Option("arg", nrr.Path). Option("num-providers", "1000"). @@ -183,7 +183,7 @@ func (t *tiros) idWorker(ctx context.Context, jobs <-chan *peer.AddrInfo, idResu var out shell.IdOutput tCtx, cancel := context.WithTimeout(ctx, 10*time.Second) - err := t.kubo.Request("id", j.ID.String()).Exec(tCtx, &out) + err := t.ipfs.Request("id", j.ID.String()).Exec(tCtx, &out) cancel() idResults <- idResult{