Merge branch 'dev'

schema.sql

@@ -389,17 +389,49 @@ CREATE TABLE `qsearch_index` (
  CONSTRAINT `qsearch_index_ibfk_2` FOREIGN KEY (`group_id`) REFERENCES `groups` (`id`) ON DELETE CASCADE
 ) ENGINE=InnoDB DEFAULT CHARSET=ascii;
 
+--
+-- Table structure for table `reports`
+--
+
+CREATE TABLE `reports` (
+ `id` int unsigned NOT NULL AUTO_INCREMENT,
+ `reporter_id` int unsigned DEFAULT NULL,
+ `target_type` tinyint unsigned NOT NULL,
+ `content_type` tinyint unsigned DEFAULT NULL,
+ `target_id` int unsigned NOT NULL,
+ `content_id` int unsigned DEFAULT NULL,
+ `comment` text NOT NULL,
+ `moderator_id` int unsigned DEFAULT NULL,
+ `time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
+ `action_time` timestamp NULL DEFAULT NULL,
+ `server_domain` varchar(100) DEFAULT NULL,
+ PRIMARY KEY (`id`),
+ KEY `reporter_id` (`reporter_id`),
+ KEY `moderator_id` (`moderator_id`),
+ CONSTRAINT `reports_ibfk_1` FOREIGN KEY (`reporter_id`) REFERENCES `users` (`id`),
+ CONSTRAINT `reports_ibfk_2` FOREIGN KEY (`moderator_id`) REFERENCES `users` (`id`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
+
 --
 -- Table structure for table `servers`
 --
 
 CREATE TABLE `servers` (
- `host` varchar(100) NOT NULL DEFAULT '',
+ `id` int unsigned NOT NULL AUTO_INCREMENT,
+ `host` varchar(100) CHARACTER SET ascii COLLATE ascii_general_ci NOT NULL,
  `software` varchar(100) DEFAULT NULL,
  `version` varchar(30) DEFAULT NULL,
- `capabilities` bigint unsigned NOT NULL DEFAULT '0',
  `last_updated` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
- PRIMARY KEY (`host`)
+ `last_error_day` date DEFAULT NULL,
+ `error_day_count` int NOT NULL DEFAULT '0',
+ `is_up` tinyint unsigned NOT NULL DEFAULT '1',
+ `is_restricted` tinyint unsigned NOT NULL DEFAULT '0',
+ `restriction` json DEFAULT NULL,
+ PRIMARY KEY (`id`),
+ UNIQUE KEY `host` (`host`),
+ KEY `is_up` (`is_up`),
+ KEY `error_day_count` (`error_day_count`),
+ KEY `is_restricted` (`is_restricted`)
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
 
 --
@@ -446,7 +478,20 @@ CREATE TABLE `signup_requests` (
  `last_name` varchar(100) DEFAULT NULL,
  `reason` text NOT NULL,
  `created_at` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
- PRIMARY KEY (`id`)
+ PRIMARY KEY (`id`),
+ UNIQUE KEY `email` (`email`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
+
+--
+-- Table structure for table `stats_daily`
+--
+
+CREATE TABLE `stats_daily` (
+ `day` date NOT NULL,
+ `type` int unsigned NOT NULL,
+ `object_id` int unsigned NOT NULL,
+ `count` int unsigned NOT NULL,
+ PRIMARY KEY (`day`,`type`,`object_id`)
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
 
 --
@@ -522,4 +567,4 @@ CREATE TABLE `wall_posts` (
 
 /*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */;
 
--- Dump completed on 2022-07-29 5:01:59
+-- Dump completed on 2023-01-20 21:59:28

src/main/java/smithereen/ApplicationContext.java

@@ -3,10 +3,12 @@
 import smithereen.activitypub.ActivityPubWorker;
 import smithereen.controllers.FriendsController;
 import smithereen.controllers.GroupsController;
+import smithereen.controllers.ModerationController;
 import smithereen.controllers.NewsfeedController;
 import smithereen.controllers.NotificationsController;
 import smithereen.controllers.ObjectLinkResolver;
 import smithereen.controllers.PrivacyController;
+import smithereen.controllers.StatsController;
 import smithereen.controllers.UserInteractionsController;
 import smithereen.controllers.UsersController;
 import smithereen.controllers.WallController;
@@ -22,6 +24,8 @@ public class ApplicationContext{
  private final PrivacyController privacyController;
  private final NewsfeedController newsfeedController;
  private final NotificationsController notificationsController;
+ private final ModerationController moderationController;
+ private final StatsController statsController;
 
  public ApplicationContext(){
  wallController=new WallController(this);
@@ -34,6 +38,8 @@ public ApplicationContext(){
  privacyController=new PrivacyController(this);
  newsfeedController=new NewsfeedController(this);
  notificationsController=new NotificationsController(this);
+ moderationController=new ModerationController(this);
+ statsController=new StatsController(this);
  }
 
  public WallController getWallController(){
@@ -75,4 +81,12 @@ public NewsfeedController getNewsfeedController(){
  public NotificationsController getNotificationsController(){
  return notificationsController;
  }
+
+ public ModerationController getModerationController(){
+ return moderationController;
+ }
+
+ public StatsController getStatsController(){
+ return statsController;
+ }
 }

src/main/java/smithereen/Config.java

@@ -70,6 +70,7 @@ public class Config{
  public static String serverAdminEmail;
  public static SignupMode signupMode=SignupMode.CLOSED;
  public static boolean signupConfirmEmail;
+ public static boolean signupFormUseCaptcha;
 
  public static String mailFrom;
  public static String smtpServerAddress;
@@ -138,6 +139,7 @@ public static void loadFromDatabase() throws SQLException{
  }catch(IllegalArgumentException ignore){}
  }
  signupConfirmEmail="1".equals(dbValues.get("SignupConfirmEmail"));
+ signupFormUseCaptcha="1".equals(dbValues.get("SignupFormUseCaptcha"));
 
  smtpServerAddress=dbValues.getOrDefault("Mail_SMTP_ServerAddress", "127.0.0.1");
  smtpPort=Utils.parseIntOrDefault(dbValues.get("Mail_SMTP_ServerPort"), 25);

src/main/java/smithereen/SmithereenApplication.java

@@ -65,6 +65,7 @@
 import spark.serialization.SerializerChain;
 import spark.utils.StringUtils;
 
+import static smithereen.Utils.randomAlphanumericString;
 import static spark.Spark.*;
 import static smithereen.sparkext.SparkExtension.*;
 
@@ -237,6 +238,14 @@ public static void main(String[] args){
  postRequiringAccessLevelWithCSRF("/users/activate", Account.AccessLevel.MODERATOR, SettingsAdminRoutes::activateAccount);
  getRequiringAccessLevel("/signupRequests", Account.AccessLevel.ADMIN, SettingsAdminRoutes::signupRequests);
  postRequiringAccessLevelWithCSRF("/signupRequests/:id/respond", Account.AccessLevel.ADMIN, SettingsAdminRoutes::respondToSignupRequest);
+ getRequiringAccessLevel("/reports", Account.AccessLevel.MODERATOR, SettingsAdminRoutes::reportsList);
+ postRequiringAccessLevelWithCSRF("/reports/:id", Account.AccessLevel.MODERATOR, SettingsAdminRoutes::reportAction);
+ postRequiringAccessLevelWithCSRF("/reports/:id/doAddCW", Account.AccessLevel.MODERATOR, SettingsAdminRoutes::reportAddCW);
+ getRequiringAccessLevel("/federation", Account.AccessLevel.MODERATOR, SettingsAdminRoutes::federationServerList);
+ getRequiringAccessLevel("/federation/:domain", Account.AccessLevel.MODERATOR, SettingsAdminRoutes::federationServerDetails);
+ getRequiringAccessLevel("/federation/:domain/restrictionForm", Account.AccessLevel.MODERATOR, SettingsAdminRoutes::federationServerRestrictionForm);
+ postRequiringAccessLevelWithCSRF("/federation/:domain/restrict", Account.AccessLevel.MODERATOR, SettingsAdminRoutes::federationRestrictServer);
+ getRequiringAccessLevelWithCSRF("/federation/:domain/resetAvailability", Account.AccessLevel.MODERATOR, SettingsAdminRoutes::federationResetServerAvailability);
  });
  });
 
@@ -279,6 +288,9 @@ public static void main(String[] args){
  getLoggedIn("/qsearch", SystemRoutes::quickSearch);
  postLoggedIn("/loadRemoteObject", SystemRoutes::loadRemoteObject);
  postWithCSRF("/votePoll", SystemRoutes::votePoll);
+ getLoggedIn("/reportForm", SystemRoutes::reportForm);
+ postWithCSRF("/submitReport", SystemRoutes::submitReport);
+ get("/captcha", SystemRoutes::captcha);
  });
 
  path("/users/:id", ()->{
@@ -408,6 +420,7 @@ public static void main(String[] args){
 
  path("/posts/:postID", ()->{
  getActivityPub("", ActivityPubRoutes::post);
+ get("/activityCreate", ActivityPubRoutes::postCreateActivity);
  get("", PostRoutes::standalonePost);
 
  getLoggedIn("/confirmDelete", PostRoutes::confirmDelete);
@@ -597,11 +610,15 @@ private static Object indexPage(Request req, Response resp){
  resp.redirect("/feed");
  return "";
  }
- return new RenderedTemplateResponse("index", req).with("title", Config.serverDisplayName)
+ RenderedTemplateResponse model=new RenderedTemplateResponse("index", req).with("title", Config.serverDisplayName)
  .with("signupMode", Config.signupMode)
  .with("serverDisplayName", Config.serverDisplayName)
  .with("serverDescription", Config.serverDescription)
  .addNavBarItem(Utils.lang(req).get("index_welcome"));
+ if((Config.signupMode==Config.SignupMode.OPEN || Config.signupMode==Config.SignupMode.MANUAL_APPROVAL) && Config.signupFormUseCaptcha){
+ model.with("captchaSid", randomAlphanumericString(16));
+ }
+ return model;
  }
 
  private static Object methodNotAllowed(Request req, Response resp){

src/main/java/smithereen/Utils.java

@@ -2,16 +2,17 @@
 
 import com.google.gson.Gson;
 import com.google.gson.GsonBuilder;
+import com.google.gson.JsonArray;
 
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
 import org.jsoup.Jsoup;
 import org.jsoup.nodes.Attribute;
-import org.jsoup.nodes.Comment;
 import org.jsoup.nodes.Document;
 import org.jsoup.nodes.Element;
 import org.jsoup.nodes.Node;
 import org.jsoup.nodes.TextNode;
+import org.jsoup.parser.Parser;
 import org.jsoup.safety.Cleaner;
 import org.jsoup.safety.Whitelist;
 import org.jsoup.select.NodeVisitor;
@@ -49,7 +50,6 @@
 import java.util.Objects;
 import java.util.Random;
 import java.util.Set;
-import java.util.TimeZone;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.TimeUnit;
 import java.util.function.Function;
@@ -63,6 +63,8 @@
 import smithereen.data.ForeignUser;
 import smithereen.data.Group;
 import smithereen.data.SessionInfo;
+import smithereen.data.StatsPoint;
+import smithereen.data.UriBuilder;
 import smithereen.data.User;
 import smithereen.data.WebDeltaResponse;
 import smithereen.exceptions.BadRequestException;
@@ -73,6 +75,8 @@
 import smithereen.storage.UserStorage;
 import smithereen.templates.RenderedTemplateResponse;
 import smithereen.util.InstantMillisJsonAdapter;
+import smithereen.util.JsonArrayBuilder;
+import smithereen.util.JsonObjectBuilder;
 import smithereen.util.LocaleJsonAdapter;
 import smithereen.util.TimeZoneJsonAdapter;
 import smithereen.util.TopLevelDomainList;
@@ -96,12 +100,17 @@ public class Utils{
  private static final String IDN_VALID_CHAR_REGEX="[[\\u00B7\\u0375\\u05F3\\u05F4\\u30FB\\u002D\\u06FD\\u06FE\\u0F0B\\u3007\\u00DF\\u03C2\\u200C\\u200D][^\\p{IsControl}\\p{IsWhite_Space}\\p{gc=S}\\p{IsPunctuation}\\p{gc=Nl}\\p{gc=No}\\p{gc=Me}\\p{blk=Combining_Diacritical_Marks}\\p{blk=Musical_Symbols}\\p{block=Ancient_Greek_Musical_Notation}\\u0640\\u07FA\\u302E\\u302F\\u3031-\\u3035\\u303B]]";
  // A domain must be at least 2 (possibly IDN) labels
  private static final String IDN_DOMAIN_REGEX=IDN_VALID_CHAR_REGEX+"+(?:\\."+IDN_VALID_CHAR_REGEX+"+)+";
- public static final Pattern URL_PATTERN=Pattern.compile("\\b(https?:\\/\\/)?("+IDN_DOMAIN_REGEX+")(?:\\:\\d+)?((?:\\/(?:[\\w\\.@%:!+-]|\\([^\\s]+?\\))*)*)(\\?(?:\\w+(?:=(?:[\\w\\.@%:!+-]|\\([^\\s]+?\\))+&?)?)+)?(#(?:[\\w\\.@%:!+-]|\\([^\\s]+?\\))+)?", Pattern.CASE_INSENSITIVE);
+ public static final Pattern URL_PATTERN=Pattern.compile("\\b(https?:\\/\\/)?("+IDN_DOMAIN_REGEX+")(?:\\:\\d+)?((?:\\/(?:[\\w\\.~@%:!+-]|\\([^\\s]+?\\))*)*)(\\?(?:\\w+(?:=(?:[\\w\\.~@%:!+-]|\\([^\\s]+?\\))+&?)?)+)?(#(?:[\\w\\.~@%:!+-]|\\([^\\s]+?\\))+)?", Pattern.CASE_INSENSITIVE);
  public static final Pattern MENTION_PATTERN=Pattern.compile("@([a-zA-Z0-9._-]+)(?:@("+IDN_DOMAIN_REGEX+"))?");
  public static final Pattern USERNAME_DOMAIN_PATTERN=Pattern.compile("@?([a-zA-Z0-9._-]+)@("+IDN_DOMAIN_REGEX+")");
  private static final Pattern SIGNATURE_HEADER_PATTERN=Pattern.compile("([!#$%^'*+\\-.^_`|~0-9A-Za-z]+)=(?:(?:\\\"((?:[^\\\"\\\\]|\\\\.)*)\\\")|([!#$%^'*+\\-.^_`|~0-9A-Za-z]+))\\s*([,;])?\\s*");
  private static final Pattern NON_ASCII_PATTERN=Pattern.compile("\\P{ASCII}");
 
+ private static final int[] GRAPH_COLORS={0x597da3, 0xb05c91, 0x4d9fab, 0x569567, 0xac4c4c, 0xc9c255, 0xcd9f4d, 0x876db3,
+ 0x6f9fc4, 0xc77bb1, 0x70c5c8, 0x80bb88, 0xce5e5e, 0xe8e282, 0xedb24a, 0xae97d3,
+ 0x6391bc, 0xc77bb1, 0x62b1bc, 0x80bb88, 0xb75454, 0xc9c255, 0xdca94f, 0x997fc4,
+ 0x85afd0, 0xc77bb1, 0x8ecfce, 0x80bb88, 0xe47070, 0xc9c255, 0xf7be5a, 0xbeaadf};
+
  public static final Gson gson=new GsonBuilder()
  .disableHtmlEscaping()
  .registerTypeAdapter(Instant.class, new InstantMillisJsonAdapter())
@@ -231,11 +240,9 @@ public static Object wrapForm(Request req, Response resp, String templateName, S
  }
 
  public static Object wrapForm(Request req, Response resp, String templateName, String formAction, String title, String buttonKey, String formID, List<String> fieldNames, Function<String, String> fieldValueGetter, String message){
- if(isAjax(req)){
+ if(isAjax(req) && StringUtils.isNotEmpty(message)){
  WebDeltaResponse wdr=new WebDeltaResponse(resp);
- if(StringUtils.isNotEmpty(message)){
- wdr.keepBox().show("formMessage_"+formID).setContent("formMessage_"+formID, escapeHTML(message));
- }
+ wdr.keepBox().show("formMessage_"+formID).setContent("formMessage_"+formID, escapeHTML(message));
  return wdr;
  }
  RenderedTemplateResponse model=new RenderedTemplateResponse(templateName, req);
@@ -326,11 +333,62 @@ public static byte[] hexStringToByteArray(String hex){
  }
 
  public static String sanitizeHTML(String src){
- return Jsoup.clean(src, HTML_SANITIZER);
+ return sanitizeHTML(src, null);
  }
 
  public static String sanitizeHTML(String src, URI documentLocation){
- return Jsoup.clean(src, documentLocation.toString(), HTML_SANITIZER);
+ Cleaner cleaner=new Cleaner(HTML_SANITIZER);
+ Document doc=Parser.parseBodyFragment(src, Objects.toString(documentLocation, ""));
+ doc.body().traverse(new NodeVisitor(){
+ private final LinkedList<ListNodeInfo> listStack=new LinkedList<>();
+
+ @Override
+ public void head(Node node, int depth){
+ if(node instanceof Element el){
+ if("ul".equals(el.tagName()) || "ol".equals(el.tagName())){
+ listStack.push(new ListNodeInfo("ol".equals(el.tagName()), el));
+ el.tagName("p");
+ }else if("li".equals(el.tagName()) && !listStack.isEmpty()){
+ ListNodeInfo info=listStack.peek();
+ String prefix=" ".repeat(listStack.size()-1);
+ if(info.isOrdered){
+ prefix+=info.currentIndex+". ";
+ info.currentIndex++;
+ }else{
+ prefix+="- ";
+ }
+ el.prependText(prefix);
+ if(el.nextSibling()!=null){
+ el.appendChild(doc.createElement("br"));
+ }
+ }else if("blockquote".equals(el.tagName())){
+ el.tagName("p");
+ el.prependText("> ");
+ }
+ }
+ }
+
+ @Override
+ public void tail(Node node, int depth){
+ if(node instanceof Element el && !listStack.isEmpty() && listStack.peek().element==el){
+ listStack.pop();
+ }
+ }
+
+ private class ListNodeInfo{
+ final boolean isOrdered;
+ final Element element;
+ int currentIndex=1;
+
+ private ListNodeInfo(boolean isOrdered, Element element){
+ this.isOrdered=isOrdered;
+ this.element=element;
+ }
+ }
+ });
+ doc.getElementsByTag("li").forEach(Element::unwrap);
+ doc.normalise();
+ return cleaner.clean(doc).body().html();
  }
 
  public static String formatDateAsISO(Instant date){
@@ -940,6 +998,18 @@ else if(value instanceof String s)
  return root.html();
  }
 
+ public static String getRequestPathAndQuery(Request req){
+ String path=req.pathInfo();
+ String query=req.queryString();
+ if(StringUtils.isNotEmpty(query)){
+ path+="?"+query;
+ if(query.contains("_ajax=1")){
+ path=new UriBuilder(path).removeQueryParam("_ajax").build().toString();
+ }
+ }
+ return path;
+ }
+
  public static InetAddress getRequestIP(Request req){
  String forwardedFor=req.headers("X-Forwarded-For");
  String ip;
@@ -955,6 +1025,30 @@ public static InetAddress getRequestIP(Request req){
  }
  }
 
+ public static JsonArray makeGraphData(List<String> categoryTitles, List<List<StatsPoint>> data, ZoneId userTimeZone){
+ if(categoryTitles.size()!=data.size())
+ throw new IllegalArgumentException("categoryTitles.size != data.size");
+ JsonArrayBuilder bldr=new JsonArrayBuilder();
+ LocalDate today=LocalDate.now(userTimeZone);
+ for(int i=0;i<categoryTitles.size();i++){
+ JsonArrayBuilder d=new JsonArrayBuilder();
+ JsonObjectBuilder obj=new JsonObjectBuilder()
+ .add("name", categoryTitles.get(i))
+ .add("c", GRAPH_COLORS[i%GRAPH_COLORS.length]);
+ for(StatsPoint pt:data.get(i)){
+ JsonArrayBuilder jpt=new JsonArrayBuilder()
+ .add(pt.date().atTime(12, 0, 0).atZone(userTimeZone).toEpochSecond())
+ .add(pt.count());
+ if(pt.date().equals(today))
+ jpt.add("-");
+ d.add(jpt);
+ }
+ obj.add("d", d);
+ bldr.add(obj);
+ }
+ return bldr.build();
+ }
+
  public interface MentionCallback{
  User resolveMention(String username, String domain);
  User resolveMention(String uri);