Skip to content

Commit

Permalink
Feat/meta data v2 (Together-Java#990)
Browse files Browse the repository at this point in the history
* alter help_thread table to add more meta data

* update write operation with extra fields

* listener that updates meta data based on thread status

* fix error in log messages

* method to update tagName on change in db

* more suitable name for class and java doc

* change name to avoid confusion & update java doc

* minor bug that would change status back to active again and log message update

* more columns added for metadata

* update listener to record added metadata columns

* spotless fix

* replacing get(0) with getFirst() for list

* add routine to settle thread status if was left open

* spotless fix

* refactor MarkHelpThreadCloseInDBRoutine and changes
* update constructor with HelpThreadLifecycleListener instance
* using HelpThreadLifecycleListener method to clean up left over threads
* increasing scope of handleArchiveStatus to package level
* adding logger to MarkHelpThreadCloseInDBRoutine class

* document updated param in MarkHelpThreadCloseInDBRoutine constructor

* use time of creation/modfication from thread and tags as csv

* rely on discord for getting thread status instead of DB

* changes
* requested changes
* update config to include tagsToIgnore
* refactor tag update logic
* update sql script
* add tagsToIgnore list to config

* change list
* adding data fields that are being collected in privacy policy
* updating duration of data storage in privacy policy doc
* HelpThreadMetadataPurger will now purge help threads data post 180 days
* instead of appending new tags, only new tags are stored now

* spotless fix

* tags from config sanitized

* requested changes

* spotless

* requested changes

* remove unnecessary map

* increase routine schedule to 24 hrs
  • Loading branch information
ankitsmt211 authored Jun 4, 2024
1 parent 9122c16 commit 7bc47d6
Show file tree
Hide file tree
Showing 8 changed files with 267 additions and 9 deletions.
9 changes: 8 additions & 1 deletion PP.md
Original file line number Diff line number Diff line change
Expand Up @@ -48,14 +48,21 @@ The databases may store
* `guild_id` of guilds the **bot** is member of (the unique id of a Discord guild),
* `channel_id` of channels belonging to guilds the **bot** is member of (the unique id of a Discord channel),
* `message_id` of messages send by users in guilds the **bot** is member of (the unique id of a Discord message),
* `participant_count` of no of people who participated in help thread discussions,
* `tags` aka categories to which these help threads belong to,
* `timestamp`s for both when thread was created and closed,
* `message_count` the no of messages that were sent in lifecycle of any help thread

_Note: Help threads are just threads that are created via forum channels, used for anyone to ask questions and get help
in certain problems._

and any combination of those.

For example, **TJ-Bot** may associate your `user_id` with a `message_id` and a `timestamp` for any message that you send in a channel belonging to guilds the **bot** is member of.

**TJ-Bot** may further store data that you explicitly provided for **TJ-Bot** to offer its services. For example the reason of a moderative action when using its moderation commands.

Furthermore, upon utilization of our help service, `user_id`s and `channel_id`s are stored to track when/how many questions a user asks. The data may be stored for up to **30** days.
Furthermore, upon utilization of our help service, `user_id`s and `channel_id`s are stored to track when/how many questions a user asks. The data may be stored for up to **180** days.

The stored data is not linked to any information that is personally identifiable.

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,9 @@
import org.togetherjava.tjbot.features.help.HelpThreadAutoArchiver;
import org.togetherjava.tjbot.features.help.HelpThreadCommand;
import org.togetherjava.tjbot.features.help.HelpThreadCreatedListener;
import org.togetherjava.tjbot.features.help.HelpThreadLifecycleListener;
import org.togetherjava.tjbot.features.help.HelpThreadMetadataPurger;
import org.togetherjava.tjbot.features.help.MarkHelpThreadCloseInDBRoutine;
import org.togetherjava.tjbot.features.help.PinnedNotificationRemover;
import org.togetherjava.tjbot.features.javamail.RSSHandlerRoutine;
import org.togetherjava.tjbot.features.jshell.JShellCommand;
Expand Down Expand Up @@ -113,6 +115,8 @@ public static Collection<Feature> createFeatures(JDA jda, Database database, Con
new CodeMessageHandler(blacklistConfig.special(), jshellEval);
ChatGptService chatGptService = new ChatGptService(config);
HelpSystemHelper helpSystemHelper = new HelpSystemHelper(config, database, chatGptService);
HelpThreadLifecycleListener helpThreadLifecycleListener =
new HelpThreadLifecycleListener(helpSystemHelper, database);

// NOTE The system can add special system relevant commands also by itself,
// hence this list may not necessarily represent the full list of all commands actually
Expand All @@ -129,6 +133,7 @@ public static Collection<Feature> createFeatures(JDA jda, Database database, Con
features.add(new HelpThreadActivityUpdater(helpSystemHelper));
features.add(new HelpThreadAutoArchiver(helpSystemHelper));
features.add(new LeftoverBookmarksCleanupRoutine(bookmarksSystem));
features.add(new MarkHelpThreadCloseInDBRoutine(database, helpThreadLifecycleListener));
features.add(new MemberCountDisplayRoutine(config));
features.add(new RSSHandlerRoutine(config, database));

Expand All @@ -151,6 +156,7 @@ public static Collection<Feature> createFeatures(JDA jda, Database database, Con
features.add(new GuildLeaveCloseThreadListener(config));
features.add(new LeftoverBookmarksListener(bookmarksSystem));
features.add(new HelpThreadCreatedListener(helpSystemHelper));
features.add(new HelpThreadLifecycleListener(helpSystemHelper, database));

// Message context commands
features.add(new TransferQuestionCommand(config, chatGptService));
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -29,11 +29,11 @@
import org.togetherjava.tjbot.features.componentids.ComponentIdInteractor;

import java.awt.Color;
import java.time.Instant;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Comparator;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Optional;
Expand Down Expand Up @@ -66,6 +66,7 @@ public final class HelpSystemHelper {
private final Set<String> categories;
private final Set<String> threadActivityTagNames;
private final String categoryRoleSuffix;

private final Database database;
private final ChatGptService chatGptService;
private static final int MAX_QUESTION_LENGTH = 200;
Expand All @@ -90,7 +91,10 @@ public HelpSystemHelper(Config config, Database database, ChatGptService chatGpt
isHelpForumName = Pattern.compile(helpForumPattern).asMatchPredicate();

List<String> categoriesList = helpConfig.getCategories();
categories = new HashSet<>(categoriesList);
categories = categoriesList.stream()
.map(String::strip)
.map(String::toLowerCase)
.collect(Collectors.toSet());
categoryRoleSuffix = helpConfig.getCategoryRoleSuffix();

Map<String, Integer> categoryToCommonDesc = IntStream.range(0, categoriesList.size())
Expand All @@ -104,6 +108,8 @@ public HelpSystemHelper(Config config, Database database, ChatGptService chatGpt
threadActivityTagNames = Arrays.stream(ThreadActivity.values())
.map(ThreadActivity::getTagName)
.collect(Collectors.toSet());


}

/**
Expand Down Expand Up @@ -221,11 +227,22 @@ private RestAction<Message> useChatGptFallbackMessage(ThreadChannel threadChanne
}

void writeHelpThreadToDatabase(long authorId, ThreadChannel threadChannel) {

Instant createdAt = threadChannel.getTimeCreated().toInstant();

String appliedTags = threadChannel.getAppliedTags()
.stream()
.filter(this::shouldIgnoreTag)
.map(ForumTag::getName)
.collect(Collectors.joining(","));

database.write(content -> {
HelpThreadsRecord helpThreadsRecord = content.newRecord(HelpThreads.HELP_THREADS)
.setAuthorId(authorId)
.setChannelId(threadChannel.getIdLong())
.setCreatedAt(threadChannel.getTimeCreated().toInstant());
.setCreatedAt(createdAt)
.setTags(appliedTags)
.setTicketStatus(TicketStatus.ACTIVE.val);
if (helpThreadsRecord.update() == 0) {
helpThreadsRecord.insert();
}
Expand Down Expand Up @@ -265,7 +282,7 @@ private Optional<ForumTag> getFirstMatchingTagOfChannel(Set<String> tagNamesToMa
ThreadChannel channel) {
return channel.getAppliedTags()
.stream()
.filter(tag -> tagNamesToMatch.contains(tag.getName()))
.filter(tag -> tagNamesToMatch.contains(tag.getName().toLowerCase()))
.min(byCategoryCommonnessAsc);
}

Expand Down Expand Up @@ -375,6 +392,17 @@ public String getTagName() {
}
}

enum TicketStatus {
ARCHIVED(0),
ACTIVE(1);

final int val;

TicketStatus(int val) {
this.val = val;
}
}

Optional<Long> getAuthorByHelpThreadId(final long channelId) {

logger.debug("Looking for thread-record using channel ID: {}", channelId);
Expand All @@ -384,4 +412,15 @@ Optional<Long> getAuthorByHelpThreadId(final long channelId) {
.where(HelpThreads.HELP_THREADS.CHANNEL_ID.eq(channelId))
.fetchOptional(HelpThreads.HELP_THREADS.AUTHOR_ID));
}


/**
* will be used to filter a tag based on categories config
*
* @param tag applied tag
* @return boolean result whether to ignore this tag or not
*/
boolean shouldIgnoreTag(ForumTag tag) {
return this.categories.contains(tag.getName().toLowerCase());
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,6 @@
import net.dv8tion.jda.api.events.message.MessageReceivedEvent;
import net.dv8tion.jda.api.hooks.ListenerAdapter;
import net.dv8tion.jda.api.requests.RestAction;
import org.jetbrains.annotations.NotNull;

import org.togetherjava.tjbot.features.EventReceiver;
import org.togetherjava.tjbot.features.UserInteractionType;
Expand Down Expand Up @@ -58,7 +57,7 @@ public HelpThreadCreatedListener(HelpSystemHelper helper) {
}

@Override
public void onMessageReceived(@NotNull MessageReceivedEvent event) {
public void onMessageReceived(MessageReceivedEvent event) {
if (event.isFromThread()) {
ThreadChannel threadChannel = event.getChannel().asThreadChannel();
Channel parentChannel = threadChannel.getParentChannel();
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,130 @@
package org.togetherjava.tjbot.features.help;

import net.dv8tion.jda.api.entities.channel.concrete.ThreadChannel;
import net.dv8tion.jda.api.entities.channel.forums.ForumTag;
import net.dv8tion.jda.api.events.channel.update.ChannelUpdateAppliedTagsEvent;
import net.dv8tion.jda.api.events.channel.update.ChannelUpdateArchivedEvent;
import net.dv8tion.jda.api.hooks.ListenerAdapter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import org.togetherjava.tjbot.db.Database;
import org.togetherjava.tjbot.features.EventReceiver;

import java.time.Instant;
import java.util.List;
import java.util.stream.Collectors;

import static org.togetherjava.tjbot.db.generated.tables.HelpThreads.HELP_THREADS;

/**
* Listens for help thread events after creation of thread. Updates metadata based on those events
* in database.
*/
public final class HelpThreadLifecycleListener extends ListenerAdapter implements EventReceiver {
private static final Logger logger = LoggerFactory.getLogger(HelpThreadLifecycleListener.class);
private final HelpSystemHelper helper;
private final Database database;

/**
* Creates a new instance.
*
* @param helper to work with the help threads
* @param database the database to store help thread metadata in
*/
public HelpThreadLifecycleListener(HelpSystemHelper helper, Database database) {
this.helper = helper;
this.database = database;
}

@Override
public void onChannelUpdateArchived(ChannelUpdateArchivedEvent event) {
ThreadChannel threadChannel = event.getChannel().asThreadChannel();

if (!helper.isHelpForumName(threadChannel.getParentChannel().getName())) {
return;
}
handleThreadStatus(threadChannel);
}

@Override
public void onChannelUpdateAppliedTags(ChannelUpdateAppliedTagsEvent event) {
ThreadChannel threadChannel = event.getChannel().asThreadChannel();

if (!helper.isHelpForumName(threadChannel.getParentChannel().getName())
|| shouldIgnoreUpdatedTagEvent(event)) {
return;
}


String newlyAppliedTagsOnly = event.getNewTags()
.stream()
.filter(helper::shouldIgnoreTag)
.map(ForumTag::getName)
.collect(Collectors.joining(","));


long threadId = threadChannel.getIdLong();

handleTagsUpdate(threadId, newlyAppliedTagsOnly);
}

private void handleThreadStatus(ThreadChannel threadChannel) {
Instant closedAt = threadChannel.getTimeArchiveInfoLastModified().toInstant();
long threadId = threadChannel.getIdLong();
boolean isArchived = threadChannel.isArchived();

if (isArchived) {
handleArchiveStatus(closedAt, threadChannel);
return;
}

updateThreadStatusToActive(threadId);
}

void handleArchiveStatus(Instant closedAt, ThreadChannel threadChannel) {
long threadId = threadChannel.getIdLong();
int messageCount = threadChannel.getMessageCount();
int participantsExceptAuthor = threadChannel.getMemberCount() - 1;

database.write(context -> context.update(HELP_THREADS)
.set(HELP_THREADS.CLOSED_AT, closedAt)
.set(HELP_THREADS.TICKET_STATUS, HelpSystemHelper.TicketStatus.ARCHIVED.val)
.set(HELP_THREADS.MESSAGE_COUNT, messageCount)
.set(HELP_THREADS.PARTICIPANTS, participantsExceptAuthor)
.where(HELP_THREADS.CHANNEL_ID.eq(threadId))
.execute());

logger.info("Thread with id: {}, updated to archived status in database", threadId);
}

private void updateThreadStatusToActive(long threadId) {
database.write(context -> context.update(HELP_THREADS)
.set(HELP_THREADS.TICKET_STATUS, HelpSystemHelper.TicketStatus.ACTIVE.val)
.where(HELP_THREADS.CHANNEL_ID.eq(threadId))
.execute());

logger.info("Thread with id: {}, updated to active status in database", threadId);
}

private void handleTagsUpdate(long threadId, String updatedTag) {
database.write(context -> context.update(HELP_THREADS)
.set(HELP_THREADS.TAGS, updatedTag)
.where(HELP_THREADS.CHANNEL_ID.eq(threadId))
.execute());

logger.info("Updated tag for thread with id: {} in database", threadId);
}

/**
* will ignore updated tag event if all new tags belong to the categories config
*
* @param event updated tags event
* @return boolean
*/
private boolean shouldIgnoreUpdatedTagEvent(ChannelUpdateAppliedTagsEvent event) {
List<ForumTag> newTags =
event.getNewTags().stream().filter(helper::shouldIgnoreTag).toList();
return newTags.isEmpty();
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@
public class HelpThreadMetadataPurger implements Routine {
private final Database database;
private static final Logger logger = LoggerFactory.getLogger(HelpThreadMetadataPurger.class);
private static final Period DELETE_MESSAGE_RECORDS_AFTER = Period.ofDays(30);
private static final Period DELETE_MESSAGE_RECORDS_AFTER = Period.ofDays(180);

/**
* Creates a new instance.
Expand All @@ -31,7 +31,7 @@ public HelpThreadMetadataPurger(Database database) {

@Override
public Schedule createSchedule() {
return new Schedule(ScheduleMode.FIXED_RATE, 0, 4, TimeUnit.HOURS);
return new Schedule(ScheduleMode.FIXED_RATE, 0, 1, TimeUnit.DAYS);
}

@Override
Expand Down
Loading

0 comments on commit 7bc47d6

Please sign in to comment.