remove the cache and filter content in post and add key word selection

packages/relay/src/routes/postRoute.ts

@@ -28,7 +28,8 @@ export default (
             }
             if (isNaN(page) || page < 1) throw Errors.INVALID_PAGE()
 
-            const posts = await postService.fetchPosts(epks, page, db)
+            const keyword = req.query.content ? req.query.content as string : undefined
+            const posts = await postService.fetchPosts(epks, page, keyword, db)
             res.json(posts)
         })
     )

packages/relay/src/routes/reportRoute.ts

@@ -6,7 +6,6 @@ import { errorHandler } from '../services/utils/ErrorHandler'
 
 import { Groth16Proof, PublicSignals } from 'snarkjs'
 import { createCheckReputationMiddleware } from '../middlewares/CheckReputationMiddleware'
-import { postService } from '../services/PostService'
 import ProofHelper from '../services/utils/ProofHelper'
 import Validator from '../services/utils/Validator'
 import { AdjudicateValue, Errors } from '../types'
@@ -36,8 +35,6 @@ export default (
             const reportId = await reportService.createReport(db, reportData)
             // 3. Adjust Post / Comment Status
             await reportService.updateObjectStatus(db, reportData)
-            // 4. Update post order
-            await postService.updateOrder(db)
             res.json({ reportId })
         })
     )

packages/relay/src/services/PostService.ts

@@ -6,7 +6,6 @@ import {
     DAY_DIFF_STAEMENT,
     DB_PATH,
     LOAD_POST_COUNT,
-    UPDATE_POST_ORDER_INTERVAL,
 } from '../config'
 import { Errors } from '../types/InternalError'
 import { Post, PostStatus } from '../types/Post'
@@ -16,23 +15,9 @@ import ProofHelper from './utils/ProofHelper'
 import TransactionManager from './utils/TransactionManager'
 
 export class PostService {
-    // TODO: modify the cache data structure to avoid memory leak
-    private cache: Post[] = []
 
-    async start(db: DB): Promise<void> {
-        const rows = await db.count('Post', {})
-        if (rows > 0) {
-            // fetch all posts during the initalization
-            await this.updateOrder(db)
-        }
-
-        // update the order of post table in the given interval
-        setInterval(async () => {
-            await this.updateOrder(db)
-        }, UPDATE_POST_ORDER_INTERVAL)
-    }
-
-    async updateOrder(db: DB): Promise<void> {
+    async fetchOrderedPosts(db: DB, offset: number, content: string | undefined): Promise<Post[]> {
+        //  TODO i think this sql is not really maintainable for long term, should consider to refactor it
         //      if user just posted, get the first ten result from db
         //      pop last one and insert new post to the first element
 
@@ -94,7 +79,10 @@ export class PostService {
                 GROUP BY
                     postId
             ) AS c ON p.postId = c.postId
-            WHERE p.status IN (${PostStatus.ON_CHAIN}, ${PostStatus.REPORTED}, ${PostStatus.DISAGREED})
+            WHERE p.status IN (${PostStatus.ON_CHAIN}, ${PostStatus.REPORTED}, ${PostStatus.DISAGREED}) 
+            
+            ${content ? `AND p.content LIKE '%${content}%'` : ''}
+            
             ORDER BY 
                 CASE
                     WHEN ${DAY_DIFF_STAEMENT} <= 2 THEN 0
@@ -109,72 +97,46 @@ export class PostService {
                     ELSE COALESCE(c.daily_comments, 0)
                 END DESC,
                 CAST(p.publishedAt AS INTEGER) DESC
+            LIMIT ${LOAD_POST_COUNT} OFFSET ${offset}
         `
         // anondb does't provide add column api
         // use lower level db api directly from postgres / sqlite
         // for the complex sql statement
         if (DB_PATH.startsWith('postgres')) {
             const pg = db as PostgresConnector
-            this.cache = (await pg.db.query(statement)).rows
+            return (await pg.db.query(statement)).rows
         } else {
             const sq = db as SQLiteConnector
-            this.cache = await sq.db.all(statement)
+            return await sq.db.all(statement)
         }
     }
 
-    private filterPostContent(post: Post): Partial<Post> {
-        if (post.status === PostStatus.ON_CHAIN) {
-            return post
-        } else if (
-            post.status === PostStatus.REPORTED ||
-            post.status === PostStatus.DISAGREED
-        ) {
-            const { content, ...restOfPost } = post
-            return restOfPost
-        }
-        return {}
-    }
-
     // returns the LOAD_POST_COUNT posts of the given page
     // page 1
     // start = (1 - 1) * LOAD_POST_COUNT = 0 ... start index
     // end = page + LOAD_POST_COUNT ... end index
-    // slice(page, end) ... the end element will be excluded
     async fetchPosts(
         epks: string[] | undefined,
         page: number,
+        keyword: string | undefined,
         db: DB
     ): Promise<Partial<Post>[] | null> {
         let posts: Post[]
         if (!epks) {
-            const start = (page - 1) * LOAD_POST_COUNT
-            if (this.cache.length == 0) {
-                const statement = `
-                    SELECT * FROM Post 
-                    WHERE status IN (${PostStatus.ON_CHAIN}, ${PostStatus.REPORTED}, ${PostStatus.DISAGREED})
-                    ORDER BY CAST(publishedAt AS INTEGER) DESC 
-                    LIMIT ${LOAD_POST_COUNT} OFFSET ${start}
-                `
-                if (DB_PATH.startsWith('postgres')) {
-                    const pg = db as PostgresConnector
-                    posts = (await pg.db.query(statement)).rows
-                } else {
-                    const sq = db as SQLiteConnector
-                    posts = await sq.db.all(statement)
-                }
-            } else {
-                posts = this.cache.slice(start, start + LOAD_POST_COUNT)
-            }
+            posts = await this.fetchOrderedPosts(db, (page - 1) * LOAD_POST_COUNT, keyword)
         } else {
+           let whereClause = {
+                epochKey: epks,
+                status: [
+                    PostStatus.ON_CHAIN,
+                    PostStatus.REPORTED,
+                    PostStatus.DISAGREED,
+                ],
+                content : keyword ? { contains: keyword } : undefined
+            }
+
             posts = await db.findMany('Post', {
-                where: {
-                    epochKey: epks,
-                    status: [
-                        PostStatus.ON_CHAIN,
-                        PostStatus.REPORTED,
-                        PostStatus.DISAGREED,
-                    ],
-                },
+                where: whereClause,
                 limit: LOAD_POST_COUNT,
             })
         }
@@ -183,11 +145,11 @@ export class PostService {
 
         return await Promise.all(
             posts.map(async (post) => {
+                // TODO should remove loop sql query and replace it as join
                 const votes = await db.findMany('Vote', {
                     where: { postId: post.postId },
                 })
-                const filteredPost = this.filterPostContent(post)
-                return { ...filteredPost, votes }
+                return { post, votes }
             })
         )
     }
@@ -214,13 +176,11 @@ export class PostService {
 
         if (!post) return null
 
-        const filteredPost = this.filterPostContent(post)
-
-        filteredPost.votes = await db.findMany('Vote', {
+        post.votes = await db.findMany('Vote', {
             where: { postId },
         })
 
-        return filteredPost
+        return post
     }
 
     async fetchMyAccountPosts(
@@ -248,11 +208,11 @@ export class PostService {
 
         return await Promise.all(
             posts.map(async (post) => {
+                // TODO should remove loop sql query and replace it as join
                 const votes = await db.findMany('Vote', {
                     where: { postId: post.postId },
                 })
-                const filteredPost = this.filterPostContent(post)
-                return { ...filteredPost, votes }
+                return { post, votes }
             })
         )
     }

packages/relay/src/services/singletons/UnirepSocialSynchronizer.ts

@@ -284,7 +284,7 @@ export class UnirepSocialSynchronizer extends Synchronizer {
         }
 
         // update post order
-        await postService.updateOrder(this.db)
+        await postService.fetchOrderedPosts(this.db)
 
         return result
     }

packages/relay/test/post.test.ts

@@ -5,7 +5,6 @@ import { UnirepApp } from '@unirep-app/contracts/typechain-types'
 import { stringifyBigInts } from '@unirep/utils'
 import { DB } from 'anondb'
 import { APP_ABI as abi } from '../src/config'
-import { postService } from '../src/services/PostService'
 import { UnirepSocialSynchronizer } from '../src/services/singletons/UnirepSocialSynchronizer'
 import IpfsHelper from '../src/services/utils/IpfsHelper'
 import { deployContracts, startServer, stopServer } from './environment'
@@ -228,8 +227,7 @@ describe('POST /post', function () {
         // insert random amount of votes into db
         await insertVotes(db)
 
-        await postService.updateOrder(db)
-
+        // since already fetch from db, no need to update order here 
         const posts = await express.get(`/api/post?page=1`).then((res) => {
             expect(res).to.have.status(200)
             return res.body
@@ -286,8 +284,6 @@ describe('POST /post', function () {
         const txHash = await post(express, userState, authentication)
         // update the cache, the amount of posts is still 10
         // since the above post is not on-chain yet
-        await postService.updateOrder(db)
-
         // one page will have 10 posts
         let posts = await express.get(`/api/post?page=1`).then((res) => {
             expect(res).to.have.status(200)
@@ -298,6 +294,7 @@ describe('POST /post', function () {
         for (let i = 0; i < post.length; i++) {
             const post = posts[i]
             expect(post.status).equal(1)
+            expect(post.content).equal('test content')
         }
 
         // second page will be empty
@@ -318,6 +315,19 @@ describe('POST /post', function () {
         expect(offChainPost.status).equal(0)
     })
 
+    it('should fetch posts by keyword', async function () {
+        let posts = await express.get(`/api/post?page=1&keyword=content`).then((res) => {
+            expect(res).to.have.status(200)
+            return res.body
+        })
+
+        for (let i = 0; i < post.length; i++) {
+            const post = posts[i]
+            expect(post.status).equal(1)
+            expect(post.content).equal('test content')
+        }
+    })
+
     it('should fetch post failed with incorrect input', async function () {
         // page number shouldn't be negative
         await express.get(`/api/post?page=-1`).then((res) => {

packages/relay/test/report.test.ts

@@ -107,10 +107,12 @@ describe('POST /api/report', function () {
                 }
             )
 
+            const testContent = 'test content'
             await express.get('/api/post/0').then((res) => {
                 expect(res).to.have.status(200)
                 const curPost = res.body as Post
                 expect(curPost.status).to.equal(1)
+                expect(curPost.content).to.equal(testContent)
             })
         }
 
@@ -173,7 +175,7 @@ describe('POST /api/report', function () {
             `/api/post/${postId}?status=${PostStatus.REPORTED}`
         )
         expect(afterReportResponse).to.have.status(200)
-        expect(afterReportResponse.body).to.not.have.property('content')
+        expect(afterReportResponse.body).equal('test content')
         expect(afterReportResponse.body).to.have.property(
             'status',
             PostStatus.REPORTED
@@ -979,7 +981,7 @@ describe('POST /api/report', function () {
         await express.get(`/api/post/${report.objectId}`).then((res) => {
             const curPost = res.body as Post
             expect(curPost.status).to.equal(PostStatus.DISAGREED)
-            expect(curPost).to.not.have.property('content')
+            expect(curPost.content).equal('test content')
         })
     })
 

packages/relay/test/vote.test.ts

@@ -7,7 +7,6 @@ import { deployContracts, startServer, stopServer } from './environment'
 
 import { UnirepApp } from '@unirep-app/contracts/typechain-types'
 import { io } from 'socket.io-client'
-import { postService } from '../src/services/PostService'
 import { UnirepSocialSynchronizer } from '../src/services/singletons/UnirepSocialSynchronizer'
 import { EventType, VoteAction, VoteMsg } from '../src/types'
 import { genAuthentication } from './utils/genAuthentication'
@@ -80,7 +79,6 @@ describe('POST /vote', function () {
             postResponses.map((txHash) => provider.waitForTransaction(txHash))
         )
         await sync.waitForSync()
-        await postService.updateOrder(db)
         // get the post ids
         const posts = await express.get('/api/post?page=1').then((res) => {
             expect(res).to.have.status(200)