diff --git a/CHANGELOG.md b/CHANGELOG.md index 212f662650..aef927c1d1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,19 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## Unreleased + +### x/data + +### Added + +[#2190](https://github.com/regen-network/regen-ledger/pull/2190) added `file-iri` query command to generate an IRI from a file. + +### Changed + +[#2190](https://github.com/regen-network/regen-ledger/pull/2190) the `convert-iri-to-hash` and `convert-hash-to-iri` query commands now work offline and don't need to talk to a node. As a result, their output is no longer wrapped in query responses, so there is a slight breaking change in the command output format. + + ## [v5.1.3](https://github.com/regen-network/regen-ledger/releases/tag/v5.1.3) - 2024-04-03 ### General diff --git a/go.mod b/go.mod index d87958d952..7adc5042a6 100644 --- a/go.mod +++ b/go.mod @@ -122,8 +122,10 @@ require ( github.com/mtibben/percent v0.2.1 // indirect github.com/pelletier/go-toml/v2 v2.0.7 // indirect github.com/petermattis/goid v0.0.0-20230317030725-371a4b8eda08 // indirect + github.com/piprate/json-gold v0.5.0 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/pquerna/cachecontrol v0.0.0-20180517163645-1555304b9b35 // indirect github.com/prometheus/client_golang v1.14.0 // indirect github.com/prometheus/client_model v0.3.0 // indirect github.com/prometheus/common v0.42.0 // indirect @@ -188,3 +190,5 @@ replace github.com/hashicorp/go-getter => github.com/hashicorp/go-getter v1.7.1 // https://github.com/regen-network/regen-ledger/security/dependabot/105 replace golang.org/x/net => golang.org/x/net v0.8.0 + +replace github.com/regen-network/regen-ledger/x/data/v2 => ./x/data diff --git a/go.sum b/go.sum index 14e3f10eb3..3e478cf54c 100644 --- a/go.sum +++ b/go.sum @@ -960,6 +960,8 @@ github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG github.com/philhofer/fwd v1.1.1/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU= github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc= github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= +github.com/piprate/json-gold v0.5.0 h1:RmGh1PYboCFcchVFuh2pbSWAZy4XJaqTMU4KQYsApbM= +github.com/piprate/json-gold v0.5.0/go.mod h1:WZ501QQMbZZ+3pXFPhQKzNwS1+jls0oqov3uQ2WasLs= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -971,6 +973,8 @@ github.com/pkg/term v0.0.0-20180730021639-bffc007b7fd5/go.mod h1:eCbImbZ95eXtAUI github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= +github.com/pquerna/cachecontrol v0.0.0-20180517163645-1555304b9b35 h1:J9b7z+QKAmPf4YLrFg6oQUotqHQeUNWwkvo7jZp1GLU= +github.com/pquerna/cachecontrol v0.0.0-20180517163645-1555304b9b35/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod h1:p2iRAGwDERtqlqzRXnrOVns+ignqQo//hLXqYxZYVNs= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= @@ -1022,8 +1026,6 @@ github.com/regen-network/regen-ledger/api/v2 v2.3.0 h1:JxIBz1TwTAxTGS+W++cNRYAix github.com/regen-network/regen-ledger/api/v2 v2.3.0/go.mod h1:acik56MozkBDT9WTnCJtOLF2gXOQfTdtgkV87ah/L5A= github.com/regen-network/regen-ledger/types/v2 v2.3.1 h1:PeO0O4xsGrIZWl6jjBlqddFIDU+wFkaUu5Ygp30PbBw= github.com/regen-network/regen-ledger/types/v2 v2.3.1/go.mod h1:h2pp1rYMAHwLuTet6XJVI6fX2lpvpiSoJVrEFfzK9n8= -github.com/regen-network/regen-ledger/x/data/v2 v2.3.1 h1:puesf8ZTtnxJ4TwZRnOs5Ksa7x/nVmvta4bpcbPwYfs= -github.com/regen-network/regen-ledger/x/data/v2 v2.3.1/go.mod h1:ndQ8oQTMbSI+1YNiqeneR87XOo2l2iXKyVbBOAN9ctQ= github.com/regen-network/regen-ledger/x/ecocredit/v3 v3.3.1 h1:vCR8s6Xt//L1JwnE8OZ7b48HsZR2InN6LMDA8KuKX78= github.com/regen-network/regen-ledger/x/ecocredit/v3 v3.3.1/go.mod h1:ZxMJW3ok1qDM4zPCvyVSE39hNg2vB3no0k8D7DTixiA= github.com/regen-network/regen-ledger/x/intertx v1.3.2 h1:KKLGAQHS7n1hBJtglOh+5R53EzyH7R3XKh/SIZMe7lU= diff --git a/x/data/client/irigen.go b/x/data/client/irigen.go new file mode 100644 index 0000000000..6d3d619ca4 --- /dev/null +++ b/x/data/client/irigen.go @@ -0,0 +1,99 @@ +package client + +import ( + "crypto" + "encoding/json" + "fmt" + "os" + "path/filepath" + + "github.com/piprate/json-gold/ld" + "github.com/spf13/cobra" + + "github.com/regen-network/regen-ledger/x/data/v2" +) + +func GenerateIRI() *cobra.Command { + return &cobra.Command{ + Use: "generate-iri [filename]", + Short: "Creates the content IRI for a file", + Long: "Creates the content IRI for a file. If the extension is .jsonld, a graph IRI will be created, otherwise a raw IRI will be created.", + Example: formatExample(`regen q data generate-iri myfile.ext`), + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + _, ctx, err := mkQueryClient(cmd) + if err != nil { + return err + } + + filename := args[0] + ext := filepath.Ext(filename) + contents, err := os.ReadFile(filename) + if err != nil { + return fmt.Errorf("failed to read file: %w", err) + } + + ch := &data.ContentHash{} + switch ext { + case ".jsonld": + proc := ld.NewJsonLdProcessor() + opts := ld.NewJsonLdOptions("") + opts.Format = "application/n-quads" + opts.Algorithm = ld.AlgorithmURDNA2015 + + var doc map[string]interface{} + err = json.Unmarshal(contents, &doc) + if err != nil { + return fmt.Errorf("failed to unmarshal json: %w", err) + } + + normalizedTriples, err := proc.Normalize(doc, opts) + if err != nil { + return fmt.Errorf("failed to normalize json: %w", err) + } + + hash, err := blake2b256hash(fmt.Sprintf("%s", normalizedTriples)) + if err != nil { + return err + } + + ch.Graph = &data.ContentHash_Graph{ + Hash: hash, + DigestAlgorithm: data.DigestAlgorithm_DIGEST_ALGORITHM_BLAKE2B_256, + CanonicalizationAlgorithm: data.GraphCanonicalizationAlgorithm_GRAPH_CANONICALIZATION_ALGORITHM_URDNA2015, + MerkleTree: data.GraphMerkleTree_GRAPH_MERKLE_TREE_NONE_UNSPECIFIED, + } + default: + hash, err := blake2b256hash(string(contents)) + if err != nil { + return err + } + + ext = ext[1:] // take the . off the extension + mediaType := data.RawMediaTypeFromExt(ext) + + ch.Raw = &data.ContentHash_Raw{ + Hash: hash, + DigestAlgorithm: data.DigestAlgorithm_DIGEST_ALGORITHM_BLAKE2B_256, + MediaType: mediaType, + } + } + + iri, err := ch.ToIRI() + if err != nil { + return fmt.Errorf("failed to convert content hash to IRI: %w", err) + } + + return ctx.PrintString(iri) + }, + } +} + +func blake2b256hash(contents string) ([]byte, error) { + hasher := crypto.BLAKE2b_256.New() + _, err := hasher.Write([]byte(contents)) + if err != nil { + return nil, fmt.Errorf("failed to hash normalized triples: %w", err) + } + return hasher.Sum(nil), nil +} diff --git a/x/data/client/query.go b/x/data/client/query.go index caea78ce4a..9bb43227a3 100644 --- a/x/data/client/query.go +++ b/x/data/client/query.go @@ -36,6 +36,7 @@ func QueryCmd(name string) *cobra.Command { QueryResolversByURLCmd(), ConvertIRIToHashCmd(), ConvertHashToIRICmd(), + GenerateIRI(), ) return cmd @@ -400,7 +401,7 @@ func QueryResolversByURLCmd() *cobra.Command { return cmd } -// ConvertIRIToHashCmd creates a CLI command for Query/ConvertIRIToHash. +// ConvertIRIToHashCmd converts an IRI to a ContentHash. func ConvertIRIToHashCmd() *cobra.Command { cmd := &cobra.Command{ Use: "convert-iri-to-hash [iri]", @@ -411,16 +412,17 @@ func ConvertIRIToHashCmd() *cobra.Command { `), Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { - c, ctx, err := mkQueryClient(cmd) + _, ctx, err := mkQueryClient(cmd) if err != nil { return err } - res, err := c.ConvertIRIToHash(cmd.Context(), &data.ConvertIRIToHashRequest{ - Iri: args[0], - }) + ch, err := data.ParseIRI(args[0]) + if err != nil { + return err + } - return printQueryResponse(ctx, res, err) + return ctx.PrintProto(ch) }, } @@ -429,7 +431,7 @@ func ConvertIRIToHashCmd() *cobra.Command { return cmd } -// ConvertHashToIRICmd creates a CLI command for Query/ConvertHashToIRI. +// ConvertHashToIRICmd converts a ContentHash to an IRI. func ConvertHashToIRICmd() *cobra.Command { cmd := &cobra.Command{ Use: "convert-hash-to-iri [hash-json]", @@ -450,7 +452,7 @@ func ConvertHashToIRICmd() *cobra.Command { `), Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { - c, ctx, err := mkQueryClient(cmd) + _, ctx, err := mkQueryClient(cmd) if err != nil { return err } @@ -460,11 +462,12 @@ func ConvertHashToIRICmd() *cobra.Command { return err } - res, err := c.ConvertHashToIRI(cmd.Context(), &data.ConvertHashToIRIRequest{ - ContentHash: contentHash, - }) + res, err := contentHash.ToIRI() + if err != nil { + return err + } - return printQueryResponse(ctx, res, err) + return ctx.PrintString(res) }, } diff --git a/x/data/client/testsuite/query.go b/x/data/client/testsuite/query.go index f2bfe804ac..bd31798978 100644 --- a/x/data/client/testsuite/query.go +++ b/x/data/client/testsuite/query.go @@ -8,6 +8,7 @@ import ( "github.com/cosmos/cosmos-sdk/testutil" "github.com/regen-network/regen-ledger/types/v2/testutil/cli" + "github.com/regen-network/regen-ledger/x/data/v2" "github.com/regen-network/regen-ledger/x/data/v2/client" ) @@ -593,9 +594,8 @@ func (s *IntegrationTestSuite) TestConvertIRIToHashCmd() { } else { require.NoError(err) - var res data.ConvertIRIToHashResponse + var res data.ContentHash require.NoError(clientCtx.Codec.UnmarshalJSON(out.Bytes(), &res)) - require.NotEmpty(res.ContentHash) } }) } @@ -650,10 +650,6 @@ func (s *IntegrationTestSuite) TestConvertHashToIRICmd() { require.Contains(out.String(), tc.expErrMsg) } else { require.NoError(err) - - var res data.ConvertHashToIRIResponse - require.NoError(clientCtx.Codec.UnmarshalJSON(out.Bytes(), &res)) - require.NotEmpty(res.Iri) } }) } diff --git a/x/data/go.mod b/x/data/go.mod index 491d0d9e8d..73d151664a 100644 --- a/x/data/go.mod +++ b/x/data/go.mod @@ -13,6 +13,7 @@ require ( github.com/golang/protobuf v1.5.3 github.com/gorilla/mux v1.8.0 github.com/grpc-ecosystem/grpc-gateway v1.16.0 + github.com/piprate/json-gold v0.5.0 github.com/regen-network/gocuke v0.6.2 github.com/regen-network/regen-ledger/api/v2 v2.3.0 github.com/regen-network/regen-ledger/types/v2 v2.3.1 @@ -128,6 +129,7 @@ require ( github.com/petermattis/goid v0.0.0-20230317030725-371a4b8eda08 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/pquerna/cachecontrol v0.0.0-20180517163645-1555304b9b35 // indirect github.com/prometheus/client_golang v1.14.0 // indirect github.com/prometheus/client_model v0.3.0 // indirect github.com/prometheus/common v0.42.0 // indirect diff --git a/x/data/go.sum b/x/data/go.sum index 6cc61a56ab..c36481f5cd 100644 --- a/x/data/go.sum +++ b/x/data/go.sum @@ -966,6 +966,8 @@ github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG github.com/philhofer/fwd v1.1.1/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU= github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc= github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= +github.com/piprate/json-gold v0.5.0 h1:RmGh1PYboCFcchVFuh2pbSWAZy4XJaqTMU4KQYsApbM= +github.com/piprate/json-gold v0.5.0/go.mod h1:WZ501QQMbZZ+3pXFPhQKzNwS1+jls0oqov3uQ2WasLs= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -977,6 +979,8 @@ github.com/pkg/term v0.0.0-20180730021639-bffc007b7fd5/go.mod h1:eCbImbZ95eXtAUI github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= +github.com/pquerna/cachecontrol v0.0.0-20180517163645-1555304b9b35 h1:J9b7z+QKAmPf4YLrFg6oQUotqHQeUNWwkvo7jZp1GLU= +github.com/pquerna/cachecontrol v0.0.0-20180517163645-1555304b9b35/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod h1:p2iRAGwDERtqlqzRXnrOVns+ignqQo//hLXqYxZYVNs= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= diff --git a/x/data/iri.go b/x/data/iri.go index a33fbd4e30..29313fc119 100644 --- a/x/data/iri.go +++ b/x/data/iri.go @@ -111,6 +111,15 @@ func init() { } } +func RawMediaTypeFromExt(ext string) RawMediaType { + mt, ok := stringToMediaExtensionType[ext] + if ok { + return mt + } + + return RawMediaType_RAW_MEDIA_TYPE_UNSPECIFIED +} + // ParseIRI parses an IRI string representation of a ContentHash into a ContentHash struct // Currently IRIs must have a "regen:" prefix, and only ContentHash_Graph and ContentHash_Raw // are supported.