From 97875ecf088138a1f329edb6ae4bc712cfec4865 Mon Sep 17 00:00:00 2001 From: FrankLi123 Date: Fri, 19 Jul 2024 19:30:13 +0800 Subject: [PATCH 01/14] feat: implement mastodon worker transform logic to handle AP object with 'Create' type (transform to activities in SocialPost and Social Comments) --- internal/engine/source/activitypub/task.go | 17 +- .../federated/activitypub/mastodon/worker.go | 148 +++++++++++++++++- 2 files changed, 160 insertions(+), 5 deletions(-) diff --git a/internal/engine/source/activitypub/task.go b/internal/engine/source/activitypub/task.go index ebabee96..13b8b90c 100644 --- a/internal/engine/source/activitypub/task.go +++ b/internal/engine/source/activitypub/task.go @@ -2,7 +2,7 @@ package activitypub import ( "fmt" - "strconv" + "time" "github.com/rss3-network/node/internal/engine" "github.com/rss3-network/node/provider/activitypub" @@ -28,7 +28,20 @@ func (t Task) GetNetwork() network.Network { func (t Task) GetTimestamp() uint64 { publishedTimeStamp := t.Message.Published - timestamp, _ := strconv.ParseUint(publishedTimeStamp, 10, 64) + + // if the Task does not have an empty 'Published' field + if publishedTimeStamp == "" { + return 0 + } + + parsedTime, err := time.Parse(time.RFC3339, publishedTimeStamp) + if err != nil { + fmt.Println("Error parsing time:", err) + return 0 + } + + // Convert the time.Time object to a Unix timestamp and cast to uint64 + timestamp := uint64(parsedTime.Unix()) return timestamp } diff --git a/internal/engine/worker/federated/activitypub/mastodon/worker.go b/internal/engine/worker/federated/activitypub/mastodon/worker.go index a50e5df7..53e4268c 100644 --- a/internal/engine/worker/federated/activitypub/mastodon/worker.go +++ b/internal/engine/worker/federated/activitypub/mastodon/worker.go @@ -2,15 +2,21 @@ package mastodon import ( "context" + "encoding/json" "fmt" + "time" "github.com/rss3-network/node/internal/engine" + source "github.com/rss3-network/node/internal/engine/source/activitypub" + "github.com/rss3-network/node/provider/activitypub" "github.com/rss3-network/node/provider/httpx" "github.com/rss3-network/node/schema/worker/federated" "github.com/rss3-network/protocol-go/schema" activityx "github.com/rss3-network/protocol-go/schema/activity" + "github.com/rss3-network/protocol-go/schema/metadata" "github.com/rss3-network/protocol-go/schema/network" "github.com/rss3-network/protocol-go/schema/tag" + "github.com/rss3-network/protocol-go/schema/typex" "go.uber.org/zap" ) @@ -41,7 +47,11 @@ func (w *worker) Tags() []tag.Tag { } func (w *worker) Types() []schema.Type { - return []schema.Type{} + return []schema.Type{ + typex.SocialComment, + typex.SocialPost, + typex.SocialShare, + } } // Filter returns a source filter. @@ -49,11 +59,143 @@ func (w *worker) Filter() engine.DataSourceFilter { return nil } -func (w *worker) Transform(_ context.Context, _ engine.Task) (*activityx.Activity, error) { +func (w *worker) Transform(ctx context.Context, task engine.Task) (*activityx.Activity, error) { zap.L().Info("[mastodon/worker.go] reached Transform()") - return nil, nil + + activityPubTask, ok := task.(*source.Task) + + if !ok { + return nil, fmt.Errorf("invalid task type: %T", task) + } + + activity, err := task.BuildActivity(activityx.WithActivityPlatform(w.Platform())) + + if err != nil { + return nil, fmt.Errorf("build activity: %w", err) + } + + // Handle ActivityPub message. + switch activityPubTask.Message.Type { + case "Create": + err := w.handleActivityPubCreate(ctx, activityPubTask.Message, activity) + if err != nil { + return nil, fmt.Errorf("error occured in handleActivityPubCreate") + } + // case "Announce": + // w.handleAnnounce(ctx, activityPubTask.Message, activity) + // case "Like": + // w.handleLike(ctx, activityPubTask.Message, activity) + default: + zap.L().Debug("unsupported type", zap.String("type", activityPubTask.Message.Type)) + } + + return activity, nil +} + +// handleActivityPubCreate handles mastodon post message. +// Currently, it only supports creation of Note object. +// Output activities are in type of 'SocialComment' and 'SocialPost' +func (w *worker) handleActivityPubCreate(ctx context.Context, message activitypub.Object, activity *activityx.Activity) error { + noteObject, ok := message.Object.(map[string]interface{}) + if !ok || noteObject["type"] != "Note" { + zap.L().Debug("unsupported object type for Create", zap.String("type", fmt.Sprintf("%T", message.Object))) + return fmt.Errorf("invalid object type for Create activity") + } + + // Convert the map to a ActivityPub Note struct + var note activitypub.Note + if err := mapToStruct(noteObject, ¬e); err != nil { + return fmt.Errorf("failed to convert note object: %w", err) + } + + // Build the Activity SocialPost object from the Note + post := w.buildPost(ctx, message, note) + activity.Type = typex.SocialPost + activity.From = message.Actor + + // Check if the Note is a reply to another post + if parentID, ok := noteObject["inReplyTo"].(string); ok { + activity.Type = typex.SocialComment + post.Target = &metadata.SocialPost{ + PublicationID: parentID, + } + } + + // Build post actions and return the result + return w.buildPostActions(ctx, []string{message.Actor}, activity, post, activity.Type) +} + +// mapToStruct converts a map to a struct using JSON marshal and unmarshal +func mapToStruct(m map[string]interface{}, v interface{}) error { + // Marshal the map to JSON + jsonData, err := json.Marshal(m) + if err != nil { + return fmt.Errorf("failed to marshal map: %w", err) + } + + // Unmarshal the JSON data into the struct + if err := json.Unmarshal(jsonData, v); err != nil { + return fmt.Errorf("failed to unmarshal into struct: %w", err) + } + + return nil +} + +// buildPost constructs a SocialPost object from ActivityPub object and note +func (w *worker) buildPost(ctx context.Context, obj activitypub.Object, note activitypub.Note) *metadata.SocialPost { + // Create a new SocialPost with the content, profile ID, publication ID, and timestamp + post := &metadata.SocialPost{ + Body: note.Content, + ProfileID: obj.Actor, + PublicationID: note.ID, + Timestamp: w.parseTimestamp(note.Published), + } + // Attach media to the post + w.buildPostMedia(ctx, post, obj.Attachment) + // w.buildPostTags(post, obj.Tag) + + return post +} + +// buildPostActions adds actions to the activity based on the post details +func (w *worker) buildPostActions(_ context.Context, actors []string, activity *activityx.Activity, post *metadata.SocialPost, socialType schema.Type) error { + // Iterate over the actors and create an action for each + for _, actor := range actors { + action := activityx.Action{ + Type: socialType, + Platform: w.Platform(), + From: actor, + To: "", + Metadata: *post, + } + activity.Actions = append(activity.Actions, &action) + } + + return nil +} + +// buildPostMedia attaches media information to the post +func (w *worker) buildPostMedia(_ context.Context, post *metadata.SocialPost, attachments []activitypub.Attachment) { + // Iterate over the attachments and add each media in attachment object to the post + for _, attachment := range attachments { + post.Media = append(post.Media, metadata.Media{ + Address: attachment.URL, + MimeType: "", // todo: MimeType is not provided in Attachement Yet + }) + } +} + +// Iterate over the attachments and add each media to the post +func (w *worker) parseTimestamp(timestamp string) uint64 { + t, err := time.Parse(time.RFC3339, timestamp) + if err != nil { + return 0 + } + + return uint64(t.Unix()) } +// NewWorker creates a new Mastodon worker instance func NewWorker() (engine.Worker, error) { httpClient, err := httpx.NewHTTPClient() From 6f97bc671ffc011e38abac4ed035aa2f05362959 Mon Sep 17 00:00:00 2001 From: FrankLi123 Date: Wed, 24 Jul 2024 17:07:46 +0800 Subject: [PATCH 02/14] feat: implement mastodon worker functions to transform AP objects ('Create','Announce', 'Like') with test cases --- internal/engine/source/activitypub/task.go | 18 +- .../federated/activitypub/mastodon/worker.go | 201 ++++++++-- .../activitypub/mastodon/worker_test.go | 348 ++++++++++++++++++ provider/activitypub/model.go | 2 + 4 files changed, 535 insertions(+), 34 deletions(-) create mode 100644 internal/engine/worker/federated/activitypub/mastodon/worker_test.go diff --git a/internal/engine/source/activitypub/task.go b/internal/engine/source/activitypub/task.go index 13b8b90c..8a950a6d 100644 --- a/internal/engine/source/activitypub/task.go +++ b/internal/engine/source/activitypub/task.go @@ -13,6 +13,8 @@ import ( var _ engine.Task = (*Task)(nil) +var defaultStartTime = "2024-07-22T00:00:00Z" + type Task struct { Network network.Network Message activitypub.Object @@ -27,23 +29,19 @@ func (t Task) GetNetwork() network.Network { } func (t Task) GetTimestamp() uint64 { - publishedTimeStamp := t.Message.Published - - // if the Task does not have an empty 'Published' field - if publishedTimeStamp == "" { - return 0 + // Use default time if Published is empty + timeStr := t.Message.Published + if timeStr == "" { + timeStr = defaultStartTime } - parsedTime, err := time.Parse(time.RFC3339, publishedTimeStamp) + parsedTime, err := time.Parse(time.RFC3339, timeStr) if err != nil { fmt.Println("Error parsing time:", err) return 0 } - // Convert the time.Time object to a Unix timestamp and cast to uint64 - timestamp := uint64(parsedTime.Unix()) - - return timestamp + return uint64(parsedTime.Unix()) } func (t Task) Validate() error { diff --git a/internal/engine/worker/federated/activitypub/mastodon/worker.go b/internal/engine/worker/federated/activitypub/mastodon/worker.go index 53e4268c..9ccaa40c 100644 --- a/internal/engine/worker/federated/activitypub/mastodon/worker.go +++ b/internal/engine/worker/federated/activitypub/mastodon/worker.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "fmt" + "strings" "time" "github.com/rss3-network/node/internal/engine" @@ -59,6 +60,7 @@ func (w *worker) Filter() engine.DataSourceFilter { return nil } +// Transform processes the task and converts it into an activity. func (w *worker) Transform(ctx context.Context, task engine.Task) (*activityx.Activity, error) { zap.L().Info("[mastodon/worker.go] reached Transform()") @@ -81,20 +83,26 @@ func (w *worker) Transform(ctx context.Context, task engine.Task) (*activityx.Ac if err != nil { return nil, fmt.Errorf("error occured in handleActivityPubCreate") } - // case "Announce": - // w.handleAnnounce(ctx, activityPubTask.Message, activity) - // case "Like": - // w.handleLike(ctx, activityPubTask.Message, activity) + case "Announce": + err := w.handleActivityPubAnnounce(ctx, activityPubTask.Message, activity) + if err != nil { + return nil, fmt.Errorf("error occurred in handleActivityPubAnnounce: %w", err) + } + case "Like": + err := w.handleActivityPubLike(ctx, activityPubTask.Message, activity) + if err != nil { + return nil, fmt.Errorf("error occurred in handleActivityPubLike: %w", err) + } default: zap.L().Debug("unsupported type", zap.String("type", activityPubTask.Message.Type)) } + zap.L().Info("Successfully transformed task", zap.String("task_id", task.ID()), zap.String("activity_id", activity.ID)) + return activity, nil } // handleActivityPubCreate handles mastodon post message. -// Currently, it only supports creation of Note object. -// Output activities are in type of 'SocialComment' and 'SocialPost' func (w *worker) handleActivityPubCreate(ctx context.Context, message activitypub.Object, activity *activityx.Activity) error { noteObject, ok := message.Object.(map[string]interface{}) if !ok || noteObject["type"] != "Note" { @@ -102,7 +110,7 @@ func (w *worker) handleActivityPubCreate(ctx context.Context, message activitypu return fmt.Errorf("invalid object type for Create activity") } - // Convert the map to a ActivityPub Note struct + // Convert the map to an ActivityPub Note struct var note activitypub.Note if err := mapToStruct(noteObject, ¬e); err != nil { return fmt.Errorf("failed to convert note object: %w", err) @@ -112,8 +120,10 @@ func (w *worker) handleActivityPubCreate(ctx context.Context, message activitypu post := w.buildPost(ctx, message, note) activity.Type = typex.SocialPost activity.From = message.Actor + activity.Platform = w.Platform() // Check if the Note is a reply to another post + // If true, then make it an activity SocialComment object if parentID, ok := noteObject["inReplyTo"].(string); ok { activity.Type = typex.SocialComment post.Target = &metadata.SocialPost{ @@ -121,8 +131,21 @@ func (w *worker) handleActivityPubCreate(ctx context.Context, message activitypu } } - // Build post actions and return the result - return w.buildPostActions(ctx, []string{message.Actor}, activity, post, activity.Type) + // Determine the main recipient of this Post + // recipient := "" + // if len(note.To) > 0 { + // recipient = note.To[0] + // } + + // Generate main action + mainAction := w.createAction(activity.Type, message.Actor, "", post) + activity.Actions = append(activity.Actions, mainAction) + + // Generate additional actions for mentions + mentionActions := w.createMentionActions(activity.Type, message.Actor, note, post) + activity.Actions = append(activity.Actions, mentionActions...) + + return nil } // mapToStruct converts a map to a struct using JSON marshal and unmarshal @@ -141,6 +164,95 @@ func mapToStruct(m map[string]interface{}, v interface{}) error { return nil } +// handleActivityPubAnnounce handles Announce activities (shares/boosts) in ActivityPub. +func (w *worker) handleActivityPubAnnounce(_ context.Context, message activitypub.Object, activity *activityx.Activity) error { + activity.Type = typex.SocialShare + activity.From = message.Actor + + // Extract object IDs from the message + objectIDs, err := extractObjectIDs(message.Object) + if err != nil { + zap.L().Debug("unsupported object type for Announce", zap.String("type", fmt.Sprintf("%T", message.Object))) + return err + } + + // Iteratively create action for every announcement of the activity + for _, announcedID := range objectIDs { + // Create a SocialPost object with the Announced ID + post := &metadata.SocialPost{ + ProfileID: message.Actor, + PublicationID: message.ID, + Timestamp: w.parseTimestamp(message.Published), + Target: &metadata.SocialPost{ + PublicationID: announcedID, + }, + } + + // Create and add action to activity + action := w.createAction(activity.Type, message.Actor, "", post) + activity.Actions = append(activity.Actions, action) + } + + return nil +} + +// handleActivityPubLike handles Like activities in ActivityPub. +func (w *worker) handleActivityPubLike(_ context.Context, message activitypub.Object, activity *activityx.Activity) error { + activity.Type = typex.SocialComment + activity.From = message.Actor + + // Extract object IDs from the message + objectIDs, err := extractObjectIDs(message.Object) + if err != nil { + zap.L().Debug("unsupported object type for Like", zap.String("type", fmt.Sprintf("%T", message.Object))) + return err + } + + for _, likedID := range objectIDs { + // Create a SocialPost object with the Liked ID + post := &metadata.SocialPost{ + ProfileID: message.Actor, + PublicationID: message.ID, + Timestamp: w.parseTimestamp(message.Published), + Target: &metadata.SocialPost{ + PublicationID: likedID, + }, + } + + // Create and add action to activity + action := w.createAction(activity.Type, message.Actor, "", post) + activity.Actions = append(activity.Actions, action) + } + + return nil +} + +// createMentionActions generates actions for mentions within a note. +func (w *worker) createMentionActions(actionType schema.Type, from string, note activitypub.Note, post *metadata.SocialPost) []*activityx.Action { + var actions []*activityx.Action + + // Make mention actions for every tag in the activity + for _, mention := range note.Tag { + if mention.Type == "Mention" { + mentionAction := w.createAction(actionType, from, mention.Href, post) + actions = append(actions, mentionAction) + } + } + + return actions +} + +// createAction creates an activity action. +func (w *worker) createAction(actionType schema.Type, from, to string, metadata metadata.Metadata) *activityx.Action { + return &activityx.Action{ + Type: actionType, + Platform: w.Platform(), + From: from, + To: to, + Metadata: metadata, + } +} + // buildPost constructs a SocialPost object from ActivityPub object and note func (w *worker) buildPost(ctx context.Context, obj activitypub.Object, note activitypub.Note) *metadata.SocialPost { // Create a new SocialPost with the content, profile ID, publication ID, and timestamp @@ -149,29 +261,24 @@ func (w *worker) buildPost(ctx context.Context, obj activitypub.Object, note act ProfileID: obj.Actor, PublicationID: note.ID, Timestamp: w.parseTimestamp(note.Published), + Handle: w.extractHandle(obj.Actor), } // Attach media to the post w.buildPostMedia(ctx, post, obj.Attachment) - // w.buildPostTags(post, obj.Tag) + w.buildPostTags(post, note.Tag) return post } -// buildPostActions adds actions to the activity based on the post details -func (w *worker) buildPostActions(_ context.Context, actors []string, activity *activityx.Activity, post *metadata.SocialPost, socialType schema.Type) error { - // Iterate over the actors and create an action for each - for _, actor := range actors { - action := activityx.Action{ - Type: socialType, - Platform: w.Platform(), - From: actor, - To: "", - Metadata: *post, - } - activity.Actions = append(activity.Actions, &action) +// extractHandle parses the username out of the actor string +func (w *worker) extractHandle(actor string) string { + // Extract the last part of the URL after the final slash + parts := strings.Split(actor, "/") + if len(parts) > 1 { + return parts[len(parts)-1] } - return nil + return actor } // buildPostMedia attaches media information to the post @@ -180,11 +287,20 @@ func (w *worker) buildPostMedia(_ context.Context, post *metadata.SocialPost, at for _, attachment := range attachments { post.Media = append(post.Media, metadata.Media{ Address: attachment.URL, - MimeType: "", // todo: MimeType is not provided in Attachement Yet + MimeType: attachment.Type, }) } } +// buildPostTags builds the Tags field in the metatdata +func (w *worker) buildPostTags(post *metadata.SocialPost, tags []activitypub.Tag) { + for _, tag := range tags { + if tag.Type == "Hashtag" { + post.Tags = append(post.Tags, tag.Name) + } + } +} + // Iterate over the attachments and add each media to the post func (w *worker) parseTimestamp(timestamp string) uint64 { t, err := time.Parse(time.RFC3339, timestamp) @@ -195,6 +311,43 @@ func (w *worker) parseTimestamp(timestamp string) uint64 { return uint64(t.Unix()) } +// extractObjectIDs used to extract Object IDs for Announce and Like ActivityPub object +func extractObjectIDs(object interface{}) ([]string, error) { + var ids []string + + switch obj := object.(type) { + case string: + ids = append(ids, obj) + case map[string]interface{}: + var nestedObject activitypub.Object + if err := mapToStruct(obj, &nestedObject); err != nil { + return nil, fmt.Errorf("failed to convert nested object: %w", err) + } + + ids = append(ids, nestedObject.ID) + case []interface{}: + for _, item := range obj { + switch item := item.(type) { + case string: + ids = append(ids, item) + case map[string]interface{}: + var nestedObject activitypub.Object + if err := mapToStruct(item, &nestedObject); err != nil { + return nil, fmt.Errorf("failed to convert nested object: %w", err) + } + + ids = append(ids, nestedObject.ID) + default: + return nil, fmt.Errorf("unsupported object type in array: %T", item) + } + } + default: + return nil, fmt.Errorf("unsupported object type: %T", obj) + } + + return ids, nil +} + // NewWorker creates a new Mastodon worker instance func NewWorker() (engine.Worker, error) { httpClient, err := httpx.NewHTTPClient() diff --git a/internal/engine/worker/federated/activitypub/mastodon/worker_test.go b/internal/engine/worker/federated/activitypub/mastodon/worker_test.go new file mode 100644 index 00000000..740d1097 --- /dev/null +++ b/internal/engine/worker/federated/activitypub/mastodon/worker_test.go @@ -0,0 +1,348 @@ +package mastodon + +import ( + "context" + "encoding/json" + "testing" + + "github.com/rss3-network/node/internal/engine/source/activitypub" + message "github.com/rss3-network/node/provider/activitypub" + "github.com/rss3-network/node/schema/worker/federated" + "github.com/rss3-network/protocol-go/schema/activity" + "github.com/rss3-network/protocol-go/schema/metadata" + "github.com/rss3-network/protocol-go/schema/network" + "github.com/rss3-network/protocol-go/schema/typex" + "github.com/stretchr/testify/require" +) + +func TestWorker(t *testing.T) { + t.Parallel() + + type arguments struct { + task *activitypub.Task + } + + testcases := []struct { + name string + arguments arguments + want *activity.Activity + wantError require.ErrorAssertionFunc + }{ + { + name: "Create A Note", + arguments: arguments{ + task: &activitypub.Task{ + Network: network.Mastodon, + Message: message.Object{ + Context: []interface{}{ + "https://www.w3.org/ns/activitystreams", + }, + ID: "https://airwaves.social/users/VicRB/statuses/112836523057177095", + Type: "Create", + Actor: "https://airwaves.social/users/VicRB", + Published: "2024-07-23T15:31:43Z", + Object: map[string]interface{}{ + "type": "Note", + "id": "https://airwaves.social/users/VicRB/statuses/112836523057177095", + "content": "

#VesselAlert #Vaixell ...

", + "published": "2024-07-23T15:31:43Z", + "to": []string{ + "https://www.w3.org/ns/activitystreams#Public", + }, + }, + }, + }, + }, + want: &activity.Activity{ + ID: "https://airwaves.social/users/VicRB/statuses/112836523057177095", + Network: network.Mastodon, + Platform: federated.PlatformMastodon.String(), + From: "https://airwaves.social/users/VicRB", + Type: typex.SocialPost, + Status: true, + Actions: []*activity.Action{ + { + Type: typex.SocialPost, + Platform: federated.PlatformMastodon.String(), + From: "https://airwaves.social/users/VicRB", + Metadata: &metadata.SocialPost{ + PublicationID: "https://airwaves.social/users/VicRB/statuses/112836523057177095", + Body: "

#VesselAlert #Vaixell ...

", + Handle: "VicRB", + ProfileID: "https://airwaves.social/users/VicRB", + Tags: nil, + Timestamp: 1721748703, + }, + }, + }, + Timestamp: 1721748703, + }, + wantError: require.NoError, + }, + { + name: "Create A Comment", + arguments: arguments{ + task: &activitypub.Task{ + Network: network.Mastodon, + Message: message.Object{ + Context: []interface{}{ + "https://www.w3.org/ns/activitystreams", + }, + ID: "https://beekeeping.ninja/users/Pagan_Animist/statuses/112840117527501203", + Type: "Create", + Published: "2024-07-24T06:45:51Z", + Actor: "https://beekeeping.ninja/users/Pagan_Animist", + Object: map[string]interface{}{ + "type": "Note", + "id": "https://beekeeping.ninja/users/Pagan_Animist/statuses/112840117527501203", + "content": "

@evedazzle

Can communities band together ...

Would they help?

I’m just thinking of your power grid and next time.

", + "inReplyTo": "https://mas.to/users/evedazzle/statuses/112802025232873362", + "published": "2024-07-24T06:45:51Z", + "to": []string{ + "https://www.w3.org/ns/activitystreams#Public", + }, + }, + }, + }, + }, + want: &activity.Activity{ + ID: "https://beekeeping.ninja/users/Pagan_Animist/statuses/112840117527501203", + Network: network.Mastodon, + Platform: federated.PlatformMastodon.String(), + From: "https://beekeeping.ninja/users/Pagan_Animist", + Type: typex.SocialComment, + Status: true, + Actions: []*activity.Action{ + { + Type: typex.SocialComment, + Platform: federated.PlatformMastodon.String(), + From: "https://beekeeping.ninja/users/Pagan_Animist", + Metadata: &metadata.SocialPost{ + PublicationID: "https://beekeeping.ninja/users/Pagan_Animist/statuses/112840117527501203", + Body: "

@evedazzle

Can communities band together ...

Would they help?

I’m just thinking of your power grid and next time.

", + Handle: "Pagan_Animist", + ProfileID: "https://beekeeping.ninja/users/Pagan_Animist", + Target: &metadata.SocialPost{ + PublicationID: "https://mas.to/users/evedazzle/statuses/112802025232873362", + }, + Timestamp: 1721803551, + }, + }, + }, + Timestamp: 1721803551, + }, + wantError: require.NoError, + }, + { + name: "Create A Comment (Includes 2 Mentions)", + arguments: arguments{ + task: &activitypub.Task{ + Network: network.Mastodon, + Message: message.Object{ + Context: []interface{}{ + "https://www.w3.org/ns/activitystreams", + }, + ID: "https://epicure.social/users/Island_Martha/statuses/112840097961400438", + Type: "Create", + Actor: "https://epicure.social/users/Island_Martha", + Published: "2024-07-24T06:40:52Z", + Object: map[string]interface{}{ + "type": "Note", + "id": "https://epicure.social/users/Island_Martha/statuses/112840097961400438", + "content": "

@SpookieRobieTheCat @eunews
Or at very least restrict or remove their voting rights. Orban is not MAGA Mike

", + "inReplyTo": "https://mastodon.social/users/SpookieRobieTheCat/statuses/112840076342641439", + "to": []string{ + "https://www.w3.org/ns/activitystreams#Public", + }, + "tag": []map[string]string{ + { + "type": "Mention", + "href": "https://mastodon.social/@SpookieRobieTheCat", + "name": "@SpookieRobieTheCat", + }, + { + "type": "Mention", + "href": "https://mastodon.social/@eunews", + "name": "@eunews", + }, + }, + }, + }, + }, + }, + want: &activity.Activity{ + ID: "https://epicure.social/users/Island_Martha/statuses/112840097961400438", + Network: network.Mastodon, + Platform: federated.PlatformMastodon.String(), + From: "https://epicure.social/users/Island_Martha", + Type: typex.SocialComment, + Status: true, + Actions: []*activity.Action{ + { + Type: typex.SocialComment, + Platform: federated.PlatformMastodon.String(), + From: "https://epicure.social/users/Island_Martha", + Metadata: &metadata.SocialPost{ + PublicationID: "https://epicure.social/users/Island_Martha/statuses/112840097961400438", + Body: "

@SpookieRobieTheCat @eunews
Or at very least restrict or remove their voting rights. Orban is not MAGA Mike

", + Handle: "Island_Martha", + ProfileID: "https://epicure.social/users/Island_Martha", + Target: &metadata.SocialPost{ + PublicationID: "https://mastodon.social/users/SpookieRobieTheCat/statuses/112840076342641439", + }, + Timestamp: 0, + }, + }, + { + Type: typex.SocialComment, + Platform: federated.PlatformMastodon.String(), + From: "https://epicure.social/users/Island_Martha", + To: "https://mastodon.social/@SpookieRobieTheCat", + Metadata: &metadata.SocialPost{ + PublicationID: "https://epicure.social/users/Island_Martha/statuses/112840097961400438", + Body: "

@SpookieRobieTheCat @eunews
Or at very least restrict or remove their voting rights. Orban is not MAGA Mike

", + Handle: "Island_Martha", + ProfileID: "https://epicure.social/users/Island_Martha", + Target: &metadata.SocialPost{ + PublicationID: "https://mastodon.social/users/SpookieRobieTheCat/statuses/112840076342641439", + }, + Timestamp: 0, + }, + }, + { + Type: typex.SocialComment, + Platform: federated.PlatformMastodon.String(), + From: "https://epicure.social/users/Island_Martha", + To: "https://mastodon.social/@eunews", + Metadata: &metadata.SocialPost{ + PublicationID: "https://epicure.social/users/Island_Martha/statuses/112840097961400438", + Body: "

@SpookieRobieTheCat @eunews
Or at very least restrict or remove their voting rights. Orban is not MAGA Mike

", + Handle: "Island_Martha", + ProfileID: "https://epicure.social/users/Island_Martha", + Target: &metadata.SocialPost{ + PublicationID: "https://mastodon.social/users/SpookieRobieTheCat/statuses/112840076342641439", + }, + Timestamp: 0, + }, + }, + }, + Timestamp: 1721803252, + }, + wantError: require.NoError, + }, + { + name: "Announce(Share)", + arguments: arguments{ + task: &activitypub.Task{ + Network: network.Mastodon, + Message: message.Object{ + Context: []interface{}{ + "https://www.w3.org/ns/activitystreams", + }, + ID: "https://relay.an.exchange/activities/d93bf6f6-832d-49d0-b841-3654d8da0b79", + Type: "Announce", + Actor: "https://relay.an.exchange/actor", + Published: "2024-07-22T00:00:00Z", + Object: map[string]interface{}{ + "type": "Note", + "id": "https://cr8r.gg/users/SarraceniaWilds#announces/112250337669855051/undo", + }, + }, + }, + }, + want: &activity.Activity{ + ID: "https://relay.an.exchange/activities/d93bf6f6-832d-49d0-b841-3654d8da0b79", + Network: network.Mastodon, + Platform: federated.PlatformMastodon.String(), + From: "https://relay.an.exchange/actor", + Type: typex.SocialShare, + Status: true, + Actions: []*activity.Action{ + { + Type: typex.SocialShare, + Platform: federated.PlatformMastodon.String(), + From: "https://relay.an.exchange/actor", + Metadata: &metadata.SocialPost{ + PublicationID: "https://relay.an.exchange/activities/d93bf6f6-832d-49d0-b841-3654d8da0b79", + ProfileID: "https://relay.an.exchange/actor", + Target: &metadata.SocialPost{ + PublicationID: "https://cr8r.gg/users/SarraceniaWilds#announces/112250337669855051/undo", + }, + Timestamp: 1721606400, + }, + }, + }, + Timestamp: 1721606400, + }, + wantError: require.NoError, + }, + { + name: "Like", + arguments: arguments{ + task: &activitypub.Task{ + Network: network.Mastodon, + Message: message.Object{ + Context: []interface{}{ + "https://www.w3.org/ns/activitystreams", + }, + ID: "https://mock.social/activities/like123", + Type: "Like", + Actor: "https://mock.social/actor", + Published: "2024-07-22T00:00:00Z", + Object: map[string]interface{}{ + "type": "Note", + "id": "https://mock.social/notes/note123", + }, + }, + }, + }, + want: &activity.Activity{ + ID: "https://mock.social/activities/like123", + Network: network.Mastodon, + Platform: federated.PlatformMastodon.String(), + From: "https://mock.social/actor", + Type: typex.SocialComment, + Status: true, + Actions: []*activity.Action{ + { + Type: typex.SocialComment, + Platform: federated.PlatformMastodon.String(), + From: "https://mock.social/actor", + Metadata: &metadata.SocialPost{ + PublicationID: "https://mock.social/activities/like123", + ProfileID: "https://mock.social/actor", + Target: &metadata.SocialPost{ + PublicationID: "https://mock.social/notes/note123", + }, + Timestamp: 1721606400, + }, + }, + }, + Timestamp: 1721606400, + }, + wantError: require.NoError, + }, + } + for _, testcase := range testcases { + testcase := testcase + + t.Run(testcase.name, func(t *testing.T) { + t.Parallel() + + ctx := context.Background() + + instance, err := NewWorker() + require.NoError(t, err) + + activity, err := instance.Transform(ctx, testcase.arguments.task) + testcase.wantError(t, err) + + data, err := json.MarshalIndent(activity, "", "\x20\x20") + require.NoError(t, err) + + t.Log(string(data)) + + require.Equal(t, testcase.want, activity) + }) + } +} diff --git a/provider/activitypub/model.go b/provider/activitypub/model.go index ac8065ce..b1cec6ce 100644 --- a/provider/activitypub/model.go +++ b/provider/activitypub/model.go @@ -31,6 +31,7 @@ type Attachment struct { type Tag struct { Type string `json:"type"` Name string `json:"name"` + Href string `json:"href"` } // Note represents a note object in ActivityPub. @@ -41,4 +42,5 @@ type Note struct { Published string `json:"published,omitempty"` To []string `json:"to,omitempty"` CC []string `json:"cc,omitempty"` + Tag []Tag `json:"tag,omitempty"` } From 53f97b915d3272ddbb4f231d4457b65ba5910ae5 Mon Sep 17 00:00:00 2001 From: FrankLi123 Date: Wed, 24 Jul 2024 18:38:37 +0800 Subject: [PATCH 03/14] fix: add TODO command for the default variable --- internal/engine/source/activitypub/task.go | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/engine/source/activitypub/task.go b/internal/engine/source/activitypub/task.go index 8a950a6d..5bba4d3b 100644 --- a/internal/engine/source/activitypub/task.go +++ b/internal/engine/source/activitypub/task.go @@ -13,6 +13,7 @@ import ( var _ engine.Task = (*Task)(nil) +// TODO: should be pulled from VSL (NetworkParams contract) var defaultStartTime = "2024-07-22T00:00:00Z" type Task struct { From 1c30a691c8dd57ec78329fee196a2db98e877756 Mon Sep 17 00:00:00 2001 From: FrankLi123 Date: Mon, 29 Jul 2024 20:56:03 +0800 Subject: [PATCH 04/14] feat: implement monitor client service for activitypub source (note: use mastodon client) --- internal/node/indexer/server.go | 5 +++ internal/node/monitor/client.go | 55 +++++++++++++++++++++++++++++++++ 2 files changed, 60 insertions(+) diff --git a/internal/node/indexer/server.go b/internal/node/indexer/server.go index c4586971..6588a397 100644 --- a/internal/node/indexer/server.go +++ b/internal/node/indexer/server.go @@ -297,6 +297,11 @@ func NewServer(ctx context.Context, config *config.Module, databaseClient databa } switch config.Network.Source() { + case network.ActivityPubSource: + instance.monitorClient, err = monitor.NewActivityPubClient(config.EndpointID, config.Parameters) + if err != nil { + return nil, fmt.Errorf("new arweave monitorClient: %w", err) + } case network.ArweaveSource: instance.monitorClient, err = monitor.NewArweaveClient() if err != nil { diff --git a/internal/node/monitor/client.go b/internal/node/monitor/client.go index 299dcdd3..c4741b78 100644 --- a/internal/node/monitor/client.go +++ b/internal/node/monitor/client.go @@ -10,6 +10,7 @@ import ( "github.com/rss3-network/node/config" "github.com/rss3-network/node/internal/node/component/rss" + "github.com/rss3-network/node/provider/activitypub/mastodon" "github.com/rss3-network/node/provider/arweave" "github.com/rss3-network/node/provider/ethereum" "github.com/rss3-network/node/provider/farcaster" @@ -25,6 +26,14 @@ type Client interface { LatestState(ctx context.Context) (uint64, uint64, error) } +// activitypubClient is a client implementation for ActivityPub. +type activitypubClient struct { + activitypubClient mastodon.Client // ToDo: (Note) Currently use Matodon Client for all ActivityPub Source. +} + +// set a default client +var _ Client = (*activitypubClient)(nil) + // ethereumClient is a client implementation for ethereum. type ethereumClient struct { ethereumClient ethereum.Client @@ -222,3 +231,49 @@ func NewRssClient(endpoint string, param *config.Parameters) (Client, error) { url: base.String(), }, nil } + +func (c *activitypubClient) CurrentState(_ CheckpointState) (uint64, uint64) { + return 0, 0 +} + +func (c *activitypubClient) TargetState(_ *config.Parameters) (uint64, uint64) { + return 0, 0 +} + +// LatestState returns the latest state of the Kafka consuming process +func (c *activitypubClient) LatestState(ctx context.Context) (uint64, uint64, error) { + // Poll the Kafka consumer to verify its working state + fetches := c.activitypubClient.GetKafkaConsumer().PollFetches(ctx) + if errs := fetches.Errors(); len(errs) > 0 { + for _, e := range errs { + fmt.Printf("consumer poll fetch error: %v\n", e.Err) + } + + return 0, 0, fmt.Errorf("consumer poll fetch error: %v", fetches.Errors()) + } + // If no errors, assume the service is healthy + return 0, 0, nil +} + +// NewActivityPubClient returns a new ActivityPub client. +func NewActivityPubClient(endpoint string, param *config.Parameters) (Client, error) { + base, err := url.Parse(endpoint) + if err != nil { + return nil, fmt.Errorf("parse ActivityPub endpoint: %w", err) + } + + // Retrieve kafkaTopic from the parameters + kafkaTopic := (*param)["mastodon_kafka_topic"].(string) + + base.Path = path.Join(base.Path, kafkaTopic) + + // Create a new activitypub(mastodon) client + mastodonClient, err := mastodon.NewClient(endpoint, kafkaTopic) + if err != nil { + return nil, fmt.Errorf("create ActivityPub client: %w", err) + } + + return &activitypubClient{ + activitypubClient: mastodonClient, + }, nil +} From 13747cda661ee72b86a0e8fb246027958cbe5650 Mon Sep 17 00:00:00 2001 From: FrankLi123 Date: Mon, 29 Jul 2024 22:14:15 +0800 Subject: [PATCH 05/14] feat: add testing cases for activitypub integration in config_test.go --- config/config_test.go | 118 +++++++++++++++++++++++++++++------------- 1 file changed, 83 insertions(+), 35 deletions(-) diff --git a/config/config_test.go b/config/config_test.go index 507932cd..4651029c 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -9,6 +9,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/rss3-network/node/schema/worker/decentralized" + "github.com/rss3-network/node/schema/worker/federated" "github.com/rss3-network/node/schema/worker/rss" "github.com/rss3-network/protocol-go/schema/network" "github.com/samber/lo" @@ -33,6 +34,10 @@ endpoints: url: https://rpc.ankr.com/eth http_headers: user-agent: rss3-node + mastodon: + url: https://30.10.000.00:9092/ + http_headers: + user-agent: rss3-node database: driver: cockroachdb partition: true @@ -67,6 +72,12 @@ component: password: pass access_key: abc access_code: def + federated: + network: mastodon + worker: mastodon + endpoint: mastodon + parameters: + mastodon_kafka_topic: activitypub_events decentralized: - network: ethereum worker: core @@ -89,6 +100,12 @@ component: "http_headers": { "user-agent": "rss3-node" } + }, + "mastodon": { + "url": "https://30.10.000.00:9092/", + "http_headers": { + "user-agent": "rss3-node" + } } }, "discovery": { @@ -147,6 +164,16 @@ component: } } ], + "federated": [ + { + "network": "mastodon", + "worker": "mastodon", + "endpoint": "mastodon", + "parameters": { + "mastodon_kafka_topic": "activitypub_events" + } + } + ], "decentralized": [ { "network": "ethereum", @@ -181,6 +208,12 @@ url = "https://rpc.ankr.com/eth" [endpoints.ethereum.http_headers] user-agent = "rss3-node" +[endpoints.mastodon] +url = "https://30.10.000.00:9092/" + + [endpoints.mastodon.http_headers] + user-agent = "rss3-node" + [discovery.server] endpoint = "https://node.mydomain.com/" global_indexer_endpoint = "https://gi.rss3.dev/" @@ -222,6 +255,14 @@ password = "pass" access_key = "abc" access_code = "def" +[[component.federated]] +network = "mastodon" +worker = "mastodon" +endpoint = "mastodon" + +[component.federated.parameters] +mastodon_kafka_topic = "activitypub_events" + [[component.decentralized]] network = "ethereum" worker = "core" @@ -253,6 +294,12 @@ var configFileExpected = &File{ "user-agent": "rss3-node", }, }, + "mastodon": { + URL: "https://30.10.000.00:9092/", + HTTPHeaders: map[string]string{ + "user-agent": "rss3-node", + }, + }, }, Discovery: &Discovery{ Operator: &Operator{ @@ -283,7 +330,22 @@ var configFileExpected = &File{ }, }, }, - Federated: nil, + Federated: []*Module{ + { + Network: network.Mastodon, + Worker: federated.Mastodon, + EndpointID: "mastodon", + Endpoint: Endpoint{ + URL: "https://30.10.000.00:9092/", + HTTPHeaders: map[string]string{ + "user-agent": "rss3-node", + }, + }, + Parameters: &Parameters{ + "mastodon_kafka_topic": "activitypub_events", + }, + }, + }, Decentralized: []*Module{ { Network: network.Ethereum, @@ -370,40 +432,6 @@ func TestSetupConfig(t *testing.T) { AssertConfig(t, f, configFileExpected) } -// func TestConfigEnvOverride(t *testing.T) { -// t.Parallel() -// -// exceptEnvironment := "testing" -// exceptDatabaseURI := "postgres://mock@localhost:26257/defaultdb" -// exceptMetricsEndpoint := "127.0.0.1:9000" -// -// t.Setenv("NODE_ENVIRONMENT", exceptEnvironment) -// t.Setenv("NODE_DATABASE_URI", exceptDatabaseURI) -// t.Setenv("NODE_OBSERVABILITY_OPENTELEMETRY_METRICS_ENDPOINT", exceptMetricsEndpoint) -// -// configDir := "/etc/rss3/node" -// fs := afero.NewMemMapFs() -// -// err := fs.Mkdir(configDir, 0o777) -// assert.NoError(t, err) -// -// file, err := fs.Create(path.Join(configDir, configName)) -// assert.NoError(t, err) -// -// _, err = file.WriteString(configExampleYaml) -// require.NoError(t, err) -// -// v := viper.New() -// v.SetFs(fs) -// -// f, err := _Setup(configName, "yaml", v) -// assert.NoError(t, err) -// -// assert.Equal(t, exceptEnvironment, f.Environment) -// assert.Equal(t, exceptDatabaseURI, f.Database.URI) -// assert.Equal(t, exceptMetricsEndpoint, f.Observability.OpenTelemetry.Metrics.Endpoint) -// } - func TestConfigFilePath(t *testing.T) { t.Parallel() @@ -500,6 +528,26 @@ func AssertConfig(t *testing.T, expect, got *File) { assert.Equal(t, expect.Discovery, got.Discovery) }) + t.Run("federated", func(t *testing.T) { + for i, federated := range expect.Component.Federated { + func(_expect, got *Module) { + t.Run(fmt.Sprintf("federated-%d", i), func(t *testing.T) { + t.Parallel() + assert.Equal(t, _expect, got) + }) + }(federated, got.Component.Federated[i]) + } + + for i, indexer := range got.Component.Federated { + func(_except, got *Module) { + t.Run(fmt.Sprintf("%s-%s", indexer.Network, indexer.Worker), func(t *testing.T) { + t.Parallel() + AssertIndexer(t, _except, got) + }) + }(configFileExpected.Component.Federated[i], indexer) + } + }) + t.Run("decentralized", func(t *testing.T) { for i, rss := range expect.Component.RSS { func(_except, got *Module) { From 5f66d3cd5c61314a0e595749c8b8e9aceb300a75 Mon Sep 17 00:00:00 2001 From: FrankLi123 Date: Sat, 10 Aug 2024 01:31:51 +0800 Subject: [PATCH 06/14] fix: resolve issues mentioned in the PR review --- config/config_test.go | 10 +++++----- .../federated/activitypub/mastodon/worker.go | 18 +++++++++--------- internal/node/indexer/server.go | 2 +- provider/activitypub/mastodon/type.go | 11 +++++++++++ 4 files changed, 26 insertions(+), 15 deletions(-) create mode 100644 provider/activitypub/mastodon/type.go diff --git a/config/config_test.go b/config/config_test.go index 4651029c..6c292c94 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -35,7 +35,7 @@ endpoints: http_headers: user-agent: rss3-node mastodon: - url: https://30.10.000.00:9092/ + url: https://0.0.0.0:9092/ http_headers: user-agent: rss3-node database: @@ -102,7 +102,7 @@ component: } }, "mastodon": { - "url": "https://30.10.000.00:9092/", + "url": "https://0.0.0.0:9092/", "http_headers": { "user-agent": "rss3-node" } @@ -209,7 +209,7 @@ url = "https://rpc.ankr.com/eth" user-agent = "rss3-node" [endpoints.mastodon] -url = "https://30.10.000.00:9092/" +url = "https://0.0.0.0:9092/" [endpoints.mastodon.http_headers] user-agent = "rss3-node" @@ -295,7 +295,7 @@ var configFileExpected = &File{ }, }, "mastodon": { - URL: "https://30.10.000.00:9092/", + URL: "https://0.0.0.0:9092/", HTTPHeaders: map[string]string{ "user-agent": "rss3-node", }, @@ -336,7 +336,7 @@ var configFileExpected = &File{ Worker: federated.Mastodon, EndpointID: "mastodon", Endpoint: Endpoint{ - URL: "https://30.10.000.00:9092/", + URL: "https://0.0.0.0:9092/", HTTPHeaders: map[string]string{ "user-agent": "rss3-node", }, diff --git a/internal/engine/worker/federated/activitypub/mastodon/worker.go b/internal/engine/worker/federated/activitypub/mastodon/worker.go index 9ccaa40c..eaa8427c 100644 --- a/internal/engine/worker/federated/activitypub/mastodon/worker.go +++ b/internal/engine/worker/federated/activitypub/mastodon/worker.go @@ -78,26 +78,26 @@ func (w *worker) Transform(ctx context.Context, task engine.Task) (*activityx.Ac // Handle ActivityPub message. switch activityPubTask.Message.Type { - case "Create": + case mastodon.MessageTypeCreate.String(): err := w.handleActivityPubCreate(ctx, activityPubTask.Message, activity) if err != nil { - return nil, fmt.Errorf("error occured in handleActivityPubCreate") + return nil, fmt.Errorf("error occurred in handleActivityPubCreate: %w", err) } - case "Announce": + case mastodon.MessageTypeAnnounce.String(): err := w.handleActivityPubAnnounce(ctx, activityPubTask.Message, activity) if err != nil { return nil, fmt.Errorf("error occurred in handleActivityPubAnnounce: %w", err) } - case "Like": + case mastodon.MessageTypeLike.String(): err := w.handleActivityPubLike(ctx, activityPubTask.Message, activity) if err != nil { return nil, fmt.Errorf("error occurred in handleActivityPubLike: %w", err) } default: - zap.L().Debug("unsupported type", zap.String("type", activityPubTask.Message.Type)) + zap.L().Info("unsupported type", zap.String("type", activityPubTask.Message.Type)) } - zap.L().Info("Successfully transformed task", zap.String("task_id", task.ID()), zap.String("activity_id", activity.ID)) + zap.L().Info("unsupported type", zap.String("type", activityPubTask.Message.Type)) return activity, nil } @@ -106,7 +106,7 @@ func (w *worker) Transform(ctx context.Context, task engine.Task) (*activityx.Ac func (w *worker) handleActivityPubCreate(ctx context.Context, message activitypub.Object, activity *activityx.Activity) error { noteObject, ok := message.Object.(map[string]interface{}) if !ok || noteObject["type"] != "Note" { - zap.L().Debug("unsupported object type for Create", zap.String("type", fmt.Sprintf("%T", message.Object))) + zap.L().Info("unsupported object type for Create", zap.String("type", fmt.Sprintf("%T", message.Object))) return fmt.Errorf("invalid object type for Create activity") } @@ -172,7 +172,7 @@ func (w *worker) handleActivityPubAnnounce(_ context.Context, message activitypu // Extract object IDs from the message objectIDs, err := extractObjectIDs(message.Object) if err != nil { - zap.L().Debug("unsupported object type for Announce", zap.String("type", fmt.Sprintf("%T", message.Object))) + zap.L().Info("unsupported object type for Announce", zap.String("type", fmt.Sprintf("%T", message.Object))) return err } @@ -204,7 +204,7 @@ func (w *worker) handleActivityPubLike(_ context.Context, message activitypub.Ob // Extract object IDs from the message objectIDs, err := extractObjectIDs(message.Object) if err != nil { - zap.L().Debug("unsupported object type for Like", zap.String("type", fmt.Sprintf("%T", message.Object))) + zap.L().Info("unsupported object type for Like", zap.String("type", fmt.Sprintf("%T", message.Object))) return err } diff --git a/internal/node/indexer/server.go b/internal/node/indexer/server.go index 6588a397..8168d677 100644 --- a/internal/node/indexer/server.go +++ b/internal/node/indexer/server.go @@ -300,7 +300,7 @@ func NewServer(ctx context.Context, config *config.Module, databaseClient databa case network.ActivityPubSource: instance.monitorClient, err = monitor.NewActivityPubClient(config.EndpointID, config.Parameters) if err != nil { - return nil, fmt.Errorf("new arweave monitorClient: %w", err) + return nil, fmt.Errorf("error occurred in creating new activitypub monitorClient: %w", err) } case network.ArweaveSource: instance.monitorClient, err = monitor.NewArweaveClient() diff --git a/provider/activitypub/mastodon/type.go b/provider/activitypub/mastodon/type.go new file mode 100644 index 00000000..b997184c --- /dev/null +++ b/provider/activitypub/mastodon/type.go @@ -0,0 +1,11 @@ +package mastodon + +//go:generate go run --mod=mod github.com/dmarkham/enumer --values --type=MessageType --output mastodon_message_type.go --transform=snake-upper +type MessageType int + +const ( + MessageTypeNone MessageType = iota // Invalid default value + MessageTypeCreate // Create ActivityPub message + MessageTypeAnnounce // Announce ActivityPub message + MessageTypeLike // Like ActivityPub message +) From 74a5468ccb049dceda6af2c83a621e0785368a47 Mon Sep 17 00:00:00 2001 From: FrankLi123 Date: Sat, 10 Aug 2024 01:34:19 +0800 Subject: [PATCH 07/14] fix: resolve linting errors --- internal/engine/worker/federated/activitypub/mastodon/worker.go | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/engine/worker/federated/activitypub/mastodon/worker.go b/internal/engine/worker/federated/activitypub/mastodon/worker.go index eaa8427c..65d20e14 100644 --- a/internal/engine/worker/federated/activitypub/mastodon/worker.go +++ b/internal/engine/worker/federated/activitypub/mastodon/worker.go @@ -10,6 +10,7 @@ import ( "github.com/rss3-network/node/internal/engine" source "github.com/rss3-network/node/internal/engine/source/activitypub" "github.com/rss3-network/node/provider/activitypub" + "github.com/rss3-network/node/provider/activitypub/mastodon" "github.com/rss3-network/node/provider/httpx" "github.com/rss3-network/node/schema/worker/federated" "github.com/rss3-network/protocol-go/schema" From 316d7513a3c6c50da0d2e440f2d6eb91a92bd122 Mon Sep 17 00:00:00 2001 From: FrankLi123 Date: Sat, 10 Aug 2024 02:26:37 +0800 Subject: [PATCH 08/14] fix: modify go-generate commands and add the type_emssage file --- go.mod | 1 + go.sum | 4 + provider/activitypub/mastodon/type.go | 2 +- provider/activitypub/mastodon/type_message.go | 90 +++++++++++++++++++ 4 files changed, 96 insertions(+), 1 deletion(-) create mode 100644 provider/activitypub/mastodon/type_message.go diff --git a/go.mod b/go.mod index 089fda79..039ef808 100644 --- a/go.mod +++ b/go.mod @@ -82,6 +82,7 @@ require ( github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/deckarep/golang-set/v2 v2.3.1 // indirect github.com/distribution/reference v0.6.0 // indirect + github.com/dmarkham/enumer v1.5.10 // indirect github.com/docker/docker v26.1.3+incompatible // indirect github.com/docker/go-connections v0.4.0 // indirect github.com/docker/go-units v0.5.0 // indirect diff --git a/go.sum b/go.sum index e0124d6e..26093b35 100644 --- a/go.sum +++ b/go.sum @@ -74,6 +74,8 @@ github.com/decred/dcrd/dcrec/secp256k1/v4 v4.3.0 h1:rpfIENRNNilwHwZeG5+P150SMrnN github.com/decred/dcrd/dcrec/secp256k1/v4 v4.3.0/go.mod h1:v57UDF4pDQJcEfFUCRop3lJL149eHGSe9Jvczhzjo/0= github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= +github.com/dmarkham/enumer v1.5.10 h1:ygL0L6quiTiH1jpp68DyvsWaea6MaZLZrTTkIS++R0M= +github.com/dmarkham/enumer v1.5.10/go.mod h1:e4VILe2b1nYK3JKJpRmNdl5xbDQvELc6tQ8b+GsGk6E= github.com/docker/docker v26.1.3+incompatible h1:lLCzRbrVZrljpVNobJu1J2FHk8V0s4BawoZippkc+xo= github.com/docker/docker v26.1.3+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ= @@ -305,6 +307,8 @@ github.com/opencontainers/image-spec v1.1.0-rc5 h1:Ygwkfw9bpDvs+c9E34SdgGOj41dX/ github.com/opencontainers/image-spec v1.1.0-rc5/go.mod h1:X4pATf0uXsnn3g5aiGIsVnJBR4mxhKzfwmvK/B2NTm8= github.com/orlangure/gnomock v0.31.0 h1:dgjlQ8DYUPMyNwMZJuYBH+/GF+e7h3sloldPzIJF4k4= github.com/orlangure/gnomock v0.31.0/go.mod h1:RagxeYv3bKi+li9Lio2Faw5t6Mcy4akkeqXzkgAS3w0= +github.com/pascaldekloe/name v1.0.0 h1:n7LKFgHixETzxpRv2R77YgPUFo85QHGZKrdaYm7eY5U= +github.com/pascaldekloe/name v1.0.0/go.mod h1:Z//MfYJnH4jVpQ9wkclwu2I2MkHmXTlT9wR5UZScttM= github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= diff --git a/provider/activitypub/mastodon/type.go b/provider/activitypub/mastodon/type.go index b997184c..190e77a9 100644 --- a/provider/activitypub/mastodon/type.go +++ b/provider/activitypub/mastodon/type.go @@ -1,6 +1,6 @@ package mastodon -//go:generate go run --mod=mod github.com/dmarkham/enumer --values --type=MessageType --output mastodon_message_type.go --transform=snake-upper +//go:generate go run --mod=mod github.com/dmarkham/enumer --values --type=MessageType --output type_message.go --transform=snake-upper type MessageType int const ( diff --git a/provider/activitypub/mastodon/type_message.go b/provider/activitypub/mastodon/type_message.go new file mode 100644 index 00000000..ea05b089 --- /dev/null +++ b/provider/activitypub/mastodon/type_message.go @@ -0,0 +1,90 @@ +// Code generated by "enumer --values --type=MessageType --output type_message.go --transform=snake-upper"; DO NOT EDIT. + +package mastodon + +import ( + "fmt" + "strings" +) + +const _MessageTypeName = "MESSAGE_TYPE_NONEMESSAGE_TYPE_CREATEMESSAGE_TYPE_ANNOUNCEMESSAGE_TYPE_LIKE" + +var _MessageTypeIndex = [...]uint8{0, 17, 36, 57, 74} + +const _MessageTypeLowerName = "message_type_nonemessage_type_createmessage_type_announcemessage_type_like" + +func (i MessageType) String() string { + if i < 0 || i >= MessageType(len(_MessageTypeIndex)-1) { + return fmt.Sprintf("MessageType(%d)", i) + } + return _MessageTypeName[_MessageTypeIndex[i]:_MessageTypeIndex[i+1]] +} + +func (MessageType) Values() []string { + return MessageTypeStrings() +} + +// An "invalid array index" compiler error signifies that the constant values have changed. +// Re-run the stringer command to generate them again. +func _MessageTypeNoOp() { + var x [1]struct{} + _ = x[MessageTypeNone-(0)] + _ = x[MessageTypeCreate-(1)] + _ = x[MessageTypeAnnounce-(2)] + _ = x[MessageTypeLike-(3)] +} + +var _MessageTypeValues = []MessageType{MessageTypeNone, MessageTypeCreate, MessageTypeAnnounce, MessageTypeLike} + +var _MessageTypeNameToValueMap = map[string]MessageType{ + _MessageTypeName[0:17]: MessageTypeNone, + _MessageTypeLowerName[0:17]: MessageTypeNone, + _MessageTypeName[17:36]: MessageTypeCreate, + _MessageTypeLowerName[17:36]: MessageTypeCreate, + _MessageTypeName[36:57]: MessageTypeAnnounce, + _MessageTypeLowerName[36:57]: MessageTypeAnnounce, + _MessageTypeName[57:74]: MessageTypeLike, + _MessageTypeLowerName[57:74]: MessageTypeLike, +} + +var _MessageTypeNames = []string{ + _MessageTypeName[0:17], + _MessageTypeName[17:36], + _MessageTypeName[36:57], + _MessageTypeName[57:74], +} + +// MessageTypeString retrieves an enum value from the enum constants string name. +// Throws an error if the param is not part of the enum. +func MessageTypeString(s string) (MessageType, error) { + if val, ok := _MessageTypeNameToValueMap[s]; ok { + return val, nil + } + + if val, ok := _MessageTypeNameToValueMap[strings.ToLower(s)]; ok { + return val, nil + } + return 0, fmt.Errorf("%s does not belong to MessageType values", s) +} + +// MessageTypeValues returns all values of the enum +func MessageTypeValues() []MessageType { + return _MessageTypeValues +} + +// MessageTypeStrings returns a slice of all String values of the enum +func MessageTypeStrings() []string { + strs := make([]string, len(_MessageTypeNames)) + copy(strs, _MessageTypeNames) + return strs +} + +// IsAMessageType returns "true" if the value is listed in the enum definition. "false" otherwise +func (i MessageType) IsAMessageType() bool { + for _, v := range _MessageTypeValues { + if i == v { + return true + } + } + return false +} From 20a45cfe8b5ddab31b45be940a016ed23a4c2ad4 Mon Sep 17 00:00:00 2001 From: FrankLi123 Date: Sat, 10 Aug 2024 02:44:49 +0800 Subject: [PATCH 09/14] fix: restructure mastodon message types --- provider/activitypub/mastodon/type_message.go | 30 +++++++++---------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/provider/activitypub/mastodon/type_message.go b/provider/activitypub/mastodon/type_message.go index ea05b089..62f34416 100644 --- a/provider/activitypub/mastodon/type_message.go +++ b/provider/activitypub/mastodon/type_message.go @@ -7,11 +7,11 @@ import ( "strings" ) -const _MessageTypeName = "MESSAGE_TYPE_NONEMESSAGE_TYPE_CREATEMESSAGE_TYPE_ANNOUNCEMESSAGE_TYPE_LIKE" +const _MessageTypeName = "NoneCreateAnnounceLike" -var _MessageTypeIndex = [...]uint8{0, 17, 36, 57, 74} +var _MessageTypeIndex = [...]uint8{0, 4, 10, 18, 22} -const _MessageTypeLowerName = "message_type_nonemessage_type_createmessage_type_announcemessage_type_like" +const _MessageTypeLowerName = "nonecreateannouncelike" func (i MessageType) String() string { if i < 0 || i >= MessageType(len(_MessageTypeIndex)-1) { @@ -37,21 +37,21 @@ func _MessageTypeNoOp() { var _MessageTypeValues = []MessageType{MessageTypeNone, MessageTypeCreate, MessageTypeAnnounce, MessageTypeLike} var _MessageTypeNameToValueMap = map[string]MessageType{ - _MessageTypeName[0:17]: MessageTypeNone, - _MessageTypeLowerName[0:17]: MessageTypeNone, - _MessageTypeName[17:36]: MessageTypeCreate, - _MessageTypeLowerName[17:36]: MessageTypeCreate, - _MessageTypeName[36:57]: MessageTypeAnnounce, - _MessageTypeLowerName[36:57]: MessageTypeAnnounce, - _MessageTypeName[57:74]: MessageTypeLike, - _MessageTypeLowerName[57:74]: MessageTypeLike, + _MessageTypeName[0:4]: MessageTypeNone, + _MessageTypeLowerName[0:4]: MessageTypeNone, + _MessageTypeName[4:10]: MessageTypeCreate, + _MessageTypeLowerName[4:10]: MessageTypeCreate, + _MessageTypeName[10:18]: MessageTypeAnnounce, + _MessageTypeLowerName[10:18]: MessageTypeAnnounce, + _MessageTypeName[18:22]: MessageTypeLike, + _MessageTypeLowerName[18:22]: MessageTypeLike, } var _MessageTypeNames = []string{ - _MessageTypeName[0:17], - _MessageTypeName[17:36], - _MessageTypeName[36:57], - _MessageTypeName[57:74], + _MessageTypeName[0:4], + _MessageTypeName[4:10], + _MessageTypeName[10:18], + _MessageTypeName[18:22], } // MessageTypeString retrieves an enum value from the enum constants string name. From adb89b160c66290ad27cc6346236bdf454234113 Mon Sep 17 00:00:00 2001 From: frank Date: Fri, 13 Sep 2024 12:06:36 +0800 Subject: [PATCH 10/14] fix: fix go generate commands --- go.mod | 1 + go.sum | 6 ++ provider/activitypub/mastodon/type.go | 15 +++- provider/activitypub/mastodon/type_message.go | 90 ------------------- 4 files changed, 21 insertions(+), 91 deletions(-) delete mode 100644 provider/activitypub/mastodon/type_message.go diff --git a/go.mod b/go.mod index 8555a6bf..2d3e5193 100644 --- a/go.mod +++ b/go.mod @@ -90,6 +90,7 @@ require ( github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/deckarep/golang-set/v2 v2.3.1 // indirect github.com/distribution/reference v0.6.0 // indirect + github.com/dmarkham/enumer v1.5.10 // indirect github.com/docker/cli v26.1.1+incompatible // indirect github.com/docker/docker v26.1.3+incompatible // indirect github.com/docker/go-connections v0.5.0 // indirect diff --git a/go.sum b/go.sum index 82be129e..49815f08 100644 --- a/go.sum +++ b/go.sum @@ -86,6 +86,8 @@ github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/r github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= +github.com/dmarkham/enumer v1.5.10 h1:ygL0L6quiTiH1jpp68DyvsWaea6MaZLZrTTkIS++R0M= +github.com/dmarkham/enumer v1.5.10/go.mod h1:e4VILe2b1nYK3JKJpRmNdl5xbDQvELc6tQ8b+GsGk6E= github.com/docker/cli v26.1.1+incompatible h1:bE1/uE2tCa08fMv+7ikLR/RDPoCqytwrLtkIkSzxLvw= github.com/docker/cli v26.1.1+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= github.com/docker/docker v26.1.3+incompatible h1:lLCzRbrVZrljpVNobJu1J2FHk8V0s4BawoZippkc+xo= @@ -349,6 +351,8 @@ github.com/orlangure/gnomock v0.31.0 h1:dgjlQ8DYUPMyNwMZJuYBH+/GF+e7h3sloldPzIJF github.com/orlangure/gnomock v0.31.0/go.mod h1:RagxeYv3bKi+li9Lio2Faw5t6Mcy4akkeqXzkgAS3w0= github.com/ory/dockertest/v3 v3.10.0 h1:4K3z2VMe8Woe++invjaTB7VRyQXQy5UY+loujO4aNE4= github.com/ory/dockertest/v3 v3.10.0/go.mod h1:nr57ZbRWMqfsdGdFNLHz5jjNdDb7VVFnzAeW1n5N1Lg= +github.com/pascaldekloe/name v1.0.0 h1:n7LKFgHixETzxpRv2R77YgPUFo85QHGZKrdaYm7eY5U= +github.com/pascaldekloe/name v1.0.0/go.mod h1:Z//MfYJnH4jVpQ9wkclwu2I2MkHmXTlT9wR5UZScttM= github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= @@ -557,6 +561,8 @@ golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.19.0 h1:fEdghXQSo20giMthA7cd28ZC+jts4amQ3YMXiP5oMQ8= +golang.org/x/mod v0.19.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= diff --git a/provider/activitypub/mastodon/type.go b/provider/activitypub/mastodon/type.go index 190e77a9..43f6edb9 100644 --- a/provider/activitypub/mastodon/type.go +++ b/provider/activitypub/mastodon/type.go @@ -1,6 +1,6 @@ package mastodon -//go:generate go run --mod=mod github.com/dmarkham/enumer --values --type=MessageType --output type_message.go --transform=snake-upper +//go:generate go run --mod=mod github.com/dmarkham/enumer --values --type=MessageType --output type_message.go --trimprefix=MessageType type MessageType int const ( @@ -9,3 +9,16 @@ const ( MessageTypeAnnounce // Announce ActivityPub message MessageTypeLike // Like ActivityPub message ) + +const ( + // ActivityPub object types + ObjectTypeNote = "Note" + + InReplyTo = "inReplyTo" + To = "to" + Type = "type" + + // ActivityPub tag types + TagTypeMention = "Mention" + TagTypeHashtag = "Hashtag" +) diff --git a/provider/activitypub/mastodon/type_message.go b/provider/activitypub/mastodon/type_message.go deleted file mode 100644 index 62f34416..00000000 --- a/provider/activitypub/mastodon/type_message.go +++ /dev/null @@ -1,90 +0,0 @@ -// Code generated by "enumer --values --type=MessageType --output type_message.go --transform=snake-upper"; DO NOT EDIT. - -package mastodon - -import ( - "fmt" - "strings" -) - -const _MessageTypeName = "NoneCreateAnnounceLike" - -var _MessageTypeIndex = [...]uint8{0, 4, 10, 18, 22} - -const _MessageTypeLowerName = "nonecreateannouncelike" - -func (i MessageType) String() string { - if i < 0 || i >= MessageType(len(_MessageTypeIndex)-1) { - return fmt.Sprintf("MessageType(%d)", i) - } - return _MessageTypeName[_MessageTypeIndex[i]:_MessageTypeIndex[i+1]] -} - -func (MessageType) Values() []string { - return MessageTypeStrings() -} - -// An "invalid array index" compiler error signifies that the constant values have changed. -// Re-run the stringer command to generate them again. -func _MessageTypeNoOp() { - var x [1]struct{} - _ = x[MessageTypeNone-(0)] - _ = x[MessageTypeCreate-(1)] - _ = x[MessageTypeAnnounce-(2)] - _ = x[MessageTypeLike-(3)] -} - -var _MessageTypeValues = []MessageType{MessageTypeNone, MessageTypeCreate, MessageTypeAnnounce, MessageTypeLike} - -var _MessageTypeNameToValueMap = map[string]MessageType{ - _MessageTypeName[0:4]: MessageTypeNone, - _MessageTypeLowerName[0:4]: MessageTypeNone, - _MessageTypeName[4:10]: MessageTypeCreate, - _MessageTypeLowerName[4:10]: MessageTypeCreate, - _MessageTypeName[10:18]: MessageTypeAnnounce, - _MessageTypeLowerName[10:18]: MessageTypeAnnounce, - _MessageTypeName[18:22]: MessageTypeLike, - _MessageTypeLowerName[18:22]: MessageTypeLike, -} - -var _MessageTypeNames = []string{ - _MessageTypeName[0:4], - _MessageTypeName[4:10], - _MessageTypeName[10:18], - _MessageTypeName[18:22], -} - -// MessageTypeString retrieves an enum value from the enum constants string name. -// Throws an error if the param is not part of the enum. -func MessageTypeString(s string) (MessageType, error) { - if val, ok := _MessageTypeNameToValueMap[s]; ok { - return val, nil - } - - if val, ok := _MessageTypeNameToValueMap[strings.ToLower(s)]; ok { - return val, nil - } - return 0, fmt.Errorf("%s does not belong to MessageType values", s) -} - -// MessageTypeValues returns all values of the enum -func MessageTypeValues() []MessageType { - return _MessageTypeValues -} - -// MessageTypeStrings returns a slice of all String values of the enum -func MessageTypeStrings() []string { - strs := make([]string, len(_MessageTypeNames)) - copy(strs, _MessageTypeNames) - return strs -} - -// IsAMessageType returns "true" if the value is listed in the enum definition. "false" otherwise -func (i MessageType) IsAMessageType() bool { - for _, v := range _MessageTypeValues { - if i == v { - return true - } - } - return false -} From deffb7c495c63077e918ea5baf80e3f616cc85df Mon Sep 17 00:00:00 2001 From: frank Date: Fri, 13 Sep 2024 12:07:08 +0800 Subject: [PATCH 11/14] fix: replace magic strings with new constants in type.go --- .../worker/federated/activitypub/mastodon/worker.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/internal/engine/worker/federated/activitypub/mastodon/worker.go b/internal/engine/worker/federated/activitypub/mastodon/worker.go index 65d20e14..382f3d2d 100644 --- a/internal/engine/worker/federated/activitypub/mastodon/worker.go +++ b/internal/engine/worker/federated/activitypub/mastodon/worker.go @@ -106,7 +106,7 @@ func (w *worker) Transform(ctx context.Context, task engine.Task) (*activityx.Ac // handleActivityPubCreate handles mastodon post message. func (w *worker) handleActivityPubCreate(ctx context.Context, message activitypub.Object, activity *activityx.Activity) error { noteObject, ok := message.Object.(map[string]interface{}) - if !ok || noteObject["type"] != "Note" { + if !ok || noteObject[mastodon.Type] != mastodon.ObjectTypeNote { zap.L().Info("unsupported object type for Create", zap.String("type", fmt.Sprintf("%T", message.Object))) return fmt.Errorf("invalid object type for Create activity") } @@ -125,7 +125,7 @@ func (w *worker) handleActivityPubCreate(ctx context.Context, message activitypu // Check if the Note is a reply to another post // If true, then make it an activity SocialComment object - if parentID, ok := noteObject["inReplyTo"].(string); ok { + if parentID, ok := noteObject[mastodon.InReplyTo].(string); ok { activity.Type = typex.SocialComment post.Target = &metadata.SocialPost{ PublicationID: parentID, @@ -234,7 +234,7 @@ func (w *worker) createMentionActions(actionType schema.Type, from string, note // Make mention actions for every tag in the activity for _, mention := range note.Tag { - if mention.Type == "Mention" { + if mention.Type == mastodon.TagTypeMention { mentionAction := w.createAction(actionType, from, mention.Href, post) actions = append(actions, mentionAction) } @@ -296,7 +296,7 @@ func (w *worker) buildPostMedia(_ context.Context, post *metadata.SocialPost, at // buildPostTags builds the Tags field in the metatdata func (w *worker) buildPostTags(post *metadata.SocialPost, tags []activitypub.Tag) { for _, tag := range tags { - if tag.Type == "Hashtag" { + if tag.Type == mastodon.TagTypeHashtag { post.Tags = append(post.Tags, tag.Name) } } From 59b4a818537e5d18e68fd3de32f8167f20080914 Mon Sep 17 00:00:00 2001 From: frank Date: Fri, 13 Sep 2024 12:59:44 +0800 Subject: [PATCH 12/14] fix: add generated type_message.go --- go.mod | 1 - go.sum | 6 -- provider/activitypub/mastodon/type_message.go | 90 +++++++++++++++++++ 3 files changed, 90 insertions(+), 7 deletions(-) create mode 100644 provider/activitypub/mastodon/type_message.go diff --git a/go.mod b/go.mod index 2d3e5193..8555a6bf 100644 --- a/go.mod +++ b/go.mod @@ -90,7 +90,6 @@ require ( github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/deckarep/golang-set/v2 v2.3.1 // indirect github.com/distribution/reference v0.6.0 // indirect - github.com/dmarkham/enumer v1.5.10 // indirect github.com/docker/cli v26.1.1+incompatible // indirect github.com/docker/docker v26.1.3+incompatible // indirect github.com/docker/go-connections v0.5.0 // indirect diff --git a/go.sum b/go.sum index 49815f08..82be129e 100644 --- a/go.sum +++ b/go.sum @@ -86,8 +86,6 @@ github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/r github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= -github.com/dmarkham/enumer v1.5.10 h1:ygL0L6quiTiH1jpp68DyvsWaea6MaZLZrTTkIS++R0M= -github.com/dmarkham/enumer v1.5.10/go.mod h1:e4VILe2b1nYK3JKJpRmNdl5xbDQvELc6tQ8b+GsGk6E= github.com/docker/cli v26.1.1+incompatible h1:bE1/uE2tCa08fMv+7ikLR/RDPoCqytwrLtkIkSzxLvw= github.com/docker/cli v26.1.1+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= github.com/docker/docker v26.1.3+incompatible h1:lLCzRbrVZrljpVNobJu1J2FHk8V0s4BawoZippkc+xo= @@ -351,8 +349,6 @@ github.com/orlangure/gnomock v0.31.0 h1:dgjlQ8DYUPMyNwMZJuYBH+/GF+e7h3sloldPzIJF github.com/orlangure/gnomock v0.31.0/go.mod h1:RagxeYv3bKi+li9Lio2Faw5t6Mcy4akkeqXzkgAS3w0= github.com/ory/dockertest/v3 v3.10.0 h1:4K3z2VMe8Woe++invjaTB7VRyQXQy5UY+loujO4aNE4= github.com/ory/dockertest/v3 v3.10.0/go.mod h1:nr57ZbRWMqfsdGdFNLHz5jjNdDb7VVFnzAeW1n5N1Lg= -github.com/pascaldekloe/name v1.0.0 h1:n7LKFgHixETzxpRv2R77YgPUFo85QHGZKrdaYm7eY5U= -github.com/pascaldekloe/name v1.0.0/go.mod h1:Z//MfYJnH4jVpQ9wkclwu2I2MkHmXTlT9wR5UZScttM= github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= @@ -561,8 +557,6 @@ golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.19.0 h1:fEdghXQSo20giMthA7cd28ZC+jts4amQ3YMXiP5oMQ8= -golang.org/x/mod v0.19.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= diff --git a/provider/activitypub/mastodon/type_message.go b/provider/activitypub/mastodon/type_message.go new file mode 100644 index 00000000..eb7400fe --- /dev/null +++ b/provider/activitypub/mastodon/type_message.go @@ -0,0 +1,90 @@ +// Code generated by "enumer --values --type=MessageType --output type_message.go --trimprefix=MessageType"; DO NOT EDIT. + +package mastodon + +import ( + "fmt" + "strings" +) + +const _MessageTypeName = "NoneCreateAnnounceLike" + +var _MessageTypeIndex = [...]uint8{0, 4, 10, 18, 22} + +const _MessageTypeLowerName = "nonecreateannouncelike" + +func (i MessageType) String() string { + if i < 0 || i >= MessageType(len(_MessageTypeIndex)-1) { + return fmt.Sprintf("MessageType(%d)", i) + } + return _MessageTypeName[_MessageTypeIndex[i]:_MessageTypeIndex[i+1]] +} + +func (MessageType) Values() []string { + return MessageTypeStrings() +} + +// An "invalid array index" compiler error signifies that the constant values have changed. +// Re-run the stringer command to generate them again. +func _MessageTypeNoOp() { + var x [1]struct{} + _ = x[MessageTypeNone-(0)] + _ = x[MessageTypeCreate-(1)] + _ = x[MessageTypeAnnounce-(2)] + _ = x[MessageTypeLike-(3)] +} + +var _MessageTypeValues = []MessageType{MessageTypeNone, MessageTypeCreate, MessageTypeAnnounce, MessageTypeLike} + +var _MessageTypeNameToValueMap = map[string]MessageType{ + _MessageTypeName[0:4]: MessageTypeNone, + _MessageTypeLowerName[0:4]: MessageTypeNone, + _MessageTypeName[4:10]: MessageTypeCreate, + _MessageTypeLowerName[4:10]: MessageTypeCreate, + _MessageTypeName[10:18]: MessageTypeAnnounce, + _MessageTypeLowerName[10:18]: MessageTypeAnnounce, + _MessageTypeName[18:22]: MessageTypeLike, + _MessageTypeLowerName[18:22]: MessageTypeLike, +} + +var _MessageTypeNames = []string{ + _MessageTypeName[0:4], + _MessageTypeName[4:10], + _MessageTypeName[10:18], + _MessageTypeName[18:22], +} + +// MessageTypeString retrieves an enum value from the enum constants string name. +// Throws an error if the param is not part of the enum. +func MessageTypeString(s string) (MessageType, error) { + if val, ok := _MessageTypeNameToValueMap[s]; ok { + return val, nil + } + + if val, ok := _MessageTypeNameToValueMap[strings.ToLower(s)]; ok { + return val, nil + } + return 0, fmt.Errorf("%s does not belong to MessageType values", s) +} + +// MessageTypeValues returns all values of the enum +func MessageTypeValues() []MessageType { + return _MessageTypeValues +} + +// MessageTypeStrings returns a slice of all String values of the enum +func MessageTypeStrings() []string { + strs := make([]string, len(_MessageTypeNames)) + copy(strs, _MessageTypeNames) + return strs +} + +// IsAMessageType returns "true" if the value is listed in the enum definition. "false" otherwise +func (i MessageType) IsAMessageType() bool { + for _, v := range _MessageTypeValues { + if i == v { + return true + } + } + return false +} From 192b0c451db95011836c2990b8878ebdabc0034f Mon Sep 17 00:00:00 2001 From: frank Date: Fri, 13 Sep 2024 13:09:42 +0800 Subject: [PATCH 13/14] fix:fix typo in module search --- cmd/main.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cmd/main.go b/cmd/main.go index dd75b79f..09487943 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -244,9 +244,9 @@ func findModuleByID(configFile *config.File, workerID string) (*config.Module, e return module, nil } - if module, found := findInComponent(configFile.Component.RSS); found { - return module, nil - } + // if module, found := findInComponent(configFile.Component.RSS); found { + // return module, nil + // } return nil, fmt.Errorf("undefined module %s", workerID) } From 49ce54f2efe23d2673de3e51870ef281da296848 Mon Sep 17 00:00:00 2001 From: frank Date: Fri, 13 Sep 2024 17:53:52 +0800 Subject: [PATCH 14/14] fix: fix typos in config_test --- config/config_test.go | 96 +++++++++++++++++++------------------------ 1 file changed, 42 insertions(+), 54 deletions(-) diff --git a/config/config_test.go b/config/config_test.go index f8c49eb1..0c38c224 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -65,16 +65,7 @@ observability: endpoint: localhost:4318 component: rss: - network: rss - worker: rsshub - endpoint: https://rsshub.app/ - parameters: - authentication: - username: user - password: pass - access_key: abc - access_code: def - - network: rss + network: rss worker: rsshub endpoint: https://rsshub.app/ parameters: @@ -162,53 +153,50 @@ component: } }, "component": { - "rss": - { - "network": "rss", - "worker": "rsshub", - "endpoint": "https://rsshub.app/", - "parameters": { - "authentication": { - "username": "user", - "password": "pass", - "access_key": "abc", - "access_code": "def" - } - } + "rss": { + "network": "rss", + "worker": "rsshub", + "endpoint": "https://rsshub.app/", + "parameters": { + "authentication": { + "username": "user", + "password": "pass", + "access_key": "abc", + "access_code": "def" } - ], - "federated": [ - { - "network": "mastodon", - "worker": "mastodon", - "endpoint": "mastodon", - "parameters": { - "mastodon_kafka_topic": "activitypub_events" - } + } + }, + "federated": [ + { + "network": "mastodon", + "worker": "mastodon", + "endpoint": "mastodon", + "parameters": { + "mastodon_kafka_topic": "activitypub_events" } - ], - }, - "decentralized": [ - { - "network": "ethereum", - "worker": "core", - "endpoint": "ethereum", - "parameters": { - "block_start": 47370106, - "block_target": 456 - } - }, - { - "network": "ethereum", - "worker": "rss3", - "endpoint": "https://rpc.ankr.com/eth", - "parameters": { - "block_start": 123, - "concurrent_block_requests": 2 - } + } + ], + "decentralized": [ + { + "network": "ethereum", + "worker": "core", + "endpoint": "ethereum", + "parameters": { + "block_start": 47370106, + "block_target": 456 } - ] - } + }, + { + "network": "ethereum", + "worker": "rss3", + "endpoint": "https://rpc.ankr.com/eth", + "parameters": { + "block_start": 123, + "concurrent_block_requests": 2 + } + } + ] +} }` configExampleToml = `environment = "development"