feat(x/data/v2/client)!: add generate-iri command

CHANGELOG.md

@@ -5,6 +5,19 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## Unreleased
+
+### x/data
+
+### Added
+
+[#2190](https://github.com/regen-network/regen-ledger/pull/2190) added `file-iri` query command to generate an IRI from a file.
+
+### Changed
+
+[#2190](https://github.com/regen-network/regen-ledger/pull/2190) the `convert-iri-to-hash` and `convert-hash-to-iri` query commands now work offline and don't need to talk to a node.
+
+
 ## [v5.1.3](https://github.com/regen-network/regen-ledger/releases/tag/v5.1.3) - 2024-04-03
 
 ### General

go.mod

@@ -122,8 +122,10 @@ require (
  github.com/mtibben/percent v0.2.1 // indirect
  github.com/pelletier/go-toml/v2 v2.0.7 // indirect
  github.com/petermattis/goid v0.0.0-20230317030725-371a4b8eda08 // indirect
+ github.com/piprate/json-gold v0.5.0 // indirect
  github.com/pkg/errors v0.9.1 // indirect
  github.com/pmezard/go-difflib v1.0.0 // indirect
+ github.com/pquerna/cachecontrol v0.0.0-20180517163645-1555304b9b35 // indirect
  github.com/prometheus/client_golang v1.14.0 // indirect
  github.com/prometheus/client_model v0.3.0 // indirect
  github.com/prometheus/common v0.42.0 // indirect
@@ -188,3 +190,5 @@ replace github.com/hashicorp/go-getter => github.com/hashicorp/go-getter v1.7.1
 
 // https://github.com/regen-network/regen-ledger/security/dependabot/105
 replace golang.org/x/net => golang.org/x/net v0.8.0
+
+replace github.com/regen-network/regen-ledger/x/data/v2 => ./x/data

go.sum

@@ -960,6 +960,8 @@ github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG
 github.com/philhofer/fwd v1.1.1/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU=
 github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc=
 github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
+github.com/piprate/json-gold v0.5.0 h1:RmGh1PYboCFcchVFuh2pbSWAZy4XJaqTMU4KQYsApbM=
+github.com/piprate/json-gold v0.5.0/go.mod h1:WZ501QQMbZZ+3pXFPhQKzNwS1+jls0oqov3uQ2WasLs=
 github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
@@ -971,6 +973,8 @@ github.com/pkg/term v0.0.0-20180730021639-bffc007b7fd5/go.mod h1:eCbImbZ95eXtAUI
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI=
+github.com/pquerna/cachecontrol v0.0.0-20180517163645-1555304b9b35 h1:J9b7z+QKAmPf4YLrFg6oQUotqHQeUNWwkvo7jZp1GLU=
+github.com/pquerna/cachecontrol v0.0.0-20180517163645-1555304b9b35/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA=
 github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
 github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod h1:p2iRAGwDERtqlqzRXnrOVns+ignqQo//hLXqYxZYVNs=
 github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
@@ -1022,8 +1026,6 @@ github.com/regen-network/regen-ledger/api/v2 v2.3.0 h1:JxIBz1TwTAxTGS+W++cNRYAix
 github.com/regen-network/regen-ledger/api/v2 v2.3.0/go.mod h1:acik56MozkBDT9WTnCJtOLF2gXOQfTdtgkV87ah/L5A=
 github.com/regen-network/regen-ledger/types/v2 v2.3.1 h1:PeO0O4xsGrIZWl6jjBlqddFIDU+wFkaUu5Ygp30PbBw=
 github.com/regen-network/regen-ledger/types/v2 v2.3.1/go.mod h1:h2pp1rYMAHwLuTet6XJVI6fX2lpvpiSoJVrEFfzK9n8=
-github.com/regen-network/regen-ledger/x/data/v2 v2.3.1 h1:puesf8ZTtnxJ4TwZRnOs5Ksa7x/nVmvta4bpcbPwYfs=
-github.com/regen-network/regen-ledger/x/data/v2 v2.3.1/go.mod h1:ndQ8oQTMbSI+1YNiqeneR87XOo2l2iXKyVbBOAN9ctQ=
 github.com/regen-network/regen-ledger/x/ecocredit/v3 v3.3.1 h1:vCR8s6Xt//L1JwnE8OZ7b48HsZR2InN6LMDA8KuKX78=
 github.com/regen-network/regen-ledger/x/ecocredit/v3 v3.3.1/go.mod h1:ZxMJW3ok1qDM4zPCvyVSE39hNg2vB3no0k8D7DTixiA=
 github.com/regen-network/regen-ledger/x/intertx v1.3.2 h1:KKLGAQHS7n1hBJtglOh+5R53EzyH7R3XKh/SIZMe7lU=

x/data/client/irigen.go

@@ -0,0 +1,100 @@
+package client
+
+import (
+ "crypto"
+ "encoding/json"
+ "fmt"
+ "os"
+ "path/filepath"
+
+ "github.com/piprate/json-gold/ld"
+ "github.com/spf13/cobra"
+
+ "github.com/regen-network/regen-ledger/x/data/v2"
+)
+
+func FileIRI() *cobra.Command {
+ return &cobra.Command{
+ Use: "file-iri [filename]",
+ Short: "Creates the content IRI for a file",
+ Long: "Creates the content IRI for a file. If the extension is .jsonld, a graph IRI will be created, otherwise a raw IRI will be created.",
+ Example: formatExample(`
+ regen q data file-iri myfile.ext`),
+ Args: cobra.ExactArgs(1),
+ RunE: func(cmd *cobra.Command, args []string) error {
+ _, ctx, err := mkQueryClient(cmd)
+ if err != nil {
+ return err
+ }
+
+ filename := args[0]
+ ext := filepath.Ext(filename)
+ contents, err := os.ReadFile(filename)
+ if err != nil {
+ return fmt.Errorf("failed to read file: %w", err)
+ }
+
+ ch := &data.ContentHash{}
+ switch ext {
+ case ".jsonld":
+ proc := ld.NewJsonLdProcessor()
+ opts := ld.NewJsonLdOptions("")
+ opts.Format = "application/n-quads"
+ opts.Algorithm = ld.AlgorithmURDNA2015
+
+ var doc map[string]interface{}
+ err = json.Unmarshal(contents, &doc)
+ if err != nil {
+ return fmt.Errorf("failed to unmarshal json: %w", err)
+ }
+
+ normalizedTriples, err := proc.Normalize(doc, opts)
+ if err != nil {
+ return fmt.Errorf("failed to normalize json: %w", err)
+ }
+
+ hash, err := blake2b256hash(fmt.Sprintf("%s", normalizedTriples))
+ if err != nil {
+ return err
+ }
+
+ ch.Graph = &data.ContentHash_Graph{
+ Hash: hash,
+ DigestAlgorithm: data.DigestAlgorithm_DIGEST_ALGORITHM_BLAKE2B_256,
+ CanonicalizationAlgorithm: data.GraphCanonicalizationAlgorithm_GRAPH_CANONICALIZATION_ALGORITHM_URDNA2015,
+ MerkleTree: data.GraphMerkleTree_GRAPH_MERKLE_TREE_NONE_UNSPECIFIED,
+ }
+ default:
+ hash, err := blake2b256hash(string(contents))
+ if err != nil {
+ return err
+ }
+
+ ext = ext[1:] // take the . off the extension
+ mediaType := data.RawMediaTypeFromExt(ext)
+
+ ch.Raw = &data.ContentHash_Raw{
+ Hash: hash,
+ DigestAlgorithm: data.DigestAlgorithm_DIGEST_ALGORITHM_BLAKE2B_256,
+ MediaType: mediaType,
+ }
+ }
+
+ iri, err := ch.ToIRI()
+ if err != nil {
+ return fmt.Errorf("failed to convert content hash to IRI: %w", err)
+ }
+
+ return ctx.PrintString(iri)
+ },
+ }
+}
+
+func blake2b256hash(contents string) ([]byte, error) {
+ hasher := crypto.BLAKE2b_256.New()
+ _, err := hasher.Write([]byte(contents))
+ if err != nil {
+ return nil, fmt.Errorf("failed to hash normalized triples: %w", err)
+ }
+ return hasher.Sum(nil), nil
+}

x/data/client/query.go

@@ -36,6 +36,7 @@ func QueryCmd(name string) *cobra.Command {
  QueryResolversByURLCmd(),
  ConvertIRIToHashCmd(),
  ConvertHashToIRICmd(),
+ FileIRI(),
  )
 
  return cmd
@@ -400,7 +401,7 @@ func QueryResolversByURLCmd() *cobra.Command {
  return cmd
 }
 
-// ConvertIRIToHashCmd creates a CLI command for Query/ConvertIRIToHash.
+// ConvertIRIToHashCmd converts an IRI to a ContentHash.
 func ConvertIRIToHashCmd() *cobra.Command {
  cmd := &cobra.Command{
  Use: "convert-iri-to-hash [iri]",
@@ -411,25 +412,24 @@ func ConvertIRIToHashCmd() *cobra.Command {
  `),
  Args: cobra.ExactArgs(1),
  RunE: func(cmd *cobra.Command, args []string) error {
- c, ctx, err := mkQueryClient(cmd)
+ _, ctx, err := mkQueryClient(cmd)
  if err != nil {
  return err
  }
 
- res, err := c.ConvertIRIToHash(cmd.Context(), &data.ConvertIRIToHashRequest{
- Iri: args[0],
- })
+ ch, err := data.ParseIRI(args[0])
+ if err != nil {
+ return err
+ }
 
- return printQueryResponse(ctx, res, err)
+ return ctx.PrintProto(ch)
  },
  }
 
- flags.AddQueryFlagsToCmd(cmd)
-
  return cmd
 }
 
-// ConvertHashToIRICmd creates a CLI command for Query/ConvertHashToIRI.
+// ConvertHashToIRICmd converts a ContentHash to an IRI.
 func ConvertHashToIRICmd() *cobra.Command {
  cmd := &cobra.Command{
  Use: "convert-hash-to-iri [hash-json]",
@@ -450,7 +450,7 @@ func ConvertHashToIRICmd() *cobra.Command {
  `),
  Args: cobra.ExactArgs(1),
  RunE: func(cmd *cobra.Command, args []string) error {
- c, ctx, err := mkQueryClient(cmd)
+ _, ctx, err := mkQueryClient(cmd)
  if err != nil {
  return err
  }
@@ -460,15 +460,14 @@ func ConvertHashToIRICmd() *cobra.Command {
  return err
  }
 
- res, err := c.ConvertHashToIRI(cmd.Context(), &data.ConvertHashToIRIRequest{
- ContentHash: contentHash,
- })
+ res, err := contentHash.ToIRI()
+ if err != nil {
+ return err
+ }
 
- return printQueryResponse(ctx, res, err)
+ return ctx.PrintString(res)
  },
  }
 
- flags.AddQueryFlagsToCmd(cmd)
-
  return cmd
 }

x/data/go.mod

@@ -13,6 +13,7 @@ require (
  github.com/golang/protobuf v1.5.3
  github.com/gorilla/mux v1.8.0
  github.com/grpc-ecosystem/grpc-gateway v1.16.0
+ github.com/piprate/json-gold v0.5.0
  github.com/regen-network/gocuke v0.6.2
  github.com/regen-network/regen-ledger/api/v2 v2.3.0
  github.com/regen-network/regen-ledger/types/v2 v2.3.1
@@ -128,6 +129,7 @@ require (
  github.com/petermattis/goid v0.0.0-20230317030725-371a4b8eda08 // indirect
  github.com/pkg/errors v0.9.1 // indirect
  github.com/pmezard/go-difflib v1.0.0 // indirect
+ github.com/pquerna/cachecontrol v0.0.0-20180517163645-1555304b9b35 // indirect
  github.com/prometheus/client_golang v1.14.0 // indirect
  github.com/prometheus/client_model v0.3.0 // indirect
  github.com/prometheus/common v0.42.0 // indirect

x/data/go.sum

@@ -966,6 +966,8 @@ github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG
 github.com/philhofer/fwd v1.1.1/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU=
 github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc=
 github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
+github.com/piprate/json-gold v0.5.0 h1:RmGh1PYboCFcchVFuh2pbSWAZy4XJaqTMU4KQYsApbM=
+github.com/piprate/json-gold v0.5.0/go.mod h1:WZ501QQMbZZ+3pXFPhQKzNwS1+jls0oqov3uQ2WasLs=
 github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
@@ -977,6 +979,8 @@ github.com/pkg/term v0.0.0-20180730021639-bffc007b7fd5/go.mod h1:eCbImbZ95eXtAUI
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI=
+github.com/pquerna/cachecontrol v0.0.0-20180517163645-1555304b9b35 h1:J9b7z+QKAmPf4YLrFg6oQUotqHQeUNWwkvo7jZp1GLU=
+github.com/pquerna/cachecontrol v0.0.0-20180517163645-1555304b9b35/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA=
 github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
 github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod h1:p2iRAGwDERtqlqzRXnrOVns+ignqQo//hLXqYxZYVNs=
 github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=

x/data/iri.go

@@ -111,6 +111,15 @@ func init() {
  }
 }
 
+func RawMediaTypeFromExt(ext string) RawMediaType {
+ mt, ok := stringToMediaExtensionType[ext]
+ if ok {
+ return mt
+ }
+
+ return RawMediaType_RAW_MEDIA_TYPE_UNSPECIFIED
+}
+
 // ParseIRI parses an IRI string representation of a ContentHash into a ContentHash struct
 // Currently IRIs must have a "regen:" prefix, and only ContentHash_Graph and ContentHash_Raw
 // are supported.