-
Notifications
You must be signed in to change notification settings - Fork 7
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Use Opensearch as the tool for searching posts #296
base: main
Are you sure you want to change the base?
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -2,13 +2,13 @@ package com.gmtkgamejam | |
|
||
import com.gmtkgamejam.koin.DatabaseModule | ||
import com.gmtkgamejam.koin.DiscordBotModule | ||
import com.gmtkgamejam.koin.SearchEngineModule | ||
import com.gmtkgamejam.routing.* | ||
import io.ktor.http.* | ||
import io.ktor.serialization.kotlinx.json.* | ||
import io.ktor.server.application.* | ||
import io.ktor.server.plugins.contentnegotiation.* | ||
import io.ktor.server.plugins.cors.* | ||
import io.ktor.server.plugins.cors.routing.CORS | ||
import io.ktor.server.plugins.cors.routing.* | ||
import kotlinx.serialization.json.Json | ||
import org.koin.core.context.startKoin | ||
import org.koin.environmentProperties | ||
|
@@ -19,16 +19,21 @@ fun main(args: Array<String>): Unit = io.ktor.server.netty.EngineMain.main(args) | |
fun Application.module() { | ||
startKoin { | ||
environmentProperties() | ||
modules(DatabaseModule, DiscordBotModule) | ||
modules( | ||
DatabaseModule, | ||
DiscordBotModule, | ||
SearchEngineModule | ||
) | ||
} | ||
|
||
configureRequestHandling() | ||
configureUserInfoRouting() | ||
configureAuthRouting() | ||
configureAdminRouting() | ||
configurePostRouting() | ||
configureFavouritesRouting() | ||
configureAuthRouting() | ||
configureDiscordBotRouting() | ||
configureFavouritesRouting() | ||
configureInfraRouting() | ||
configurePostRouting() | ||
configureRequestHandling() | ||
configureUserInfoRouting() | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. The |
||
} | ||
|
||
fun Application.configureRequestHandling() { | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,21 @@ | ||
package com.gmtkgamejam | ||
|
||
import com.auth0.jwt.JWT | ||
import com.gmtkgamejam.models.auth.AuthTokenSet | ||
import com.gmtkgamejam.services.AuthService | ||
import io.ktor.http.* | ||
import io.ktor.server.application.* | ||
import io.ktor.server.request.* | ||
import io.ktor.server.response.* | ||
|
||
fun ApplicationCall.getAuthTokenSet(authService: AuthService): AuthTokenSet? { | ||
return this.request.header("Authorization") | ||
?.substringAfter("Bearer ") | ||
?.let { JWT.decode(it) }?.getClaim("id")?.asString() | ||
?.let { authService.getTokenSet(it) } | ||
} | ||
|
||
suspend fun ApplicationCall.respondJSON(text: String, status: HttpStatusCode? = null) { | ||
status?.let { response.status(it) } | ||
respond(mapOf("message" to text)) | ||
} |
This file was deleted.
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,10 +1,5 @@ | ||
package com.gmtkgamejam | ||
|
||
/** | ||
* Floating function to cast a String to an Enum without throwing an exception | ||
* | ||
* Suggest using with mapNotNull{} where possible | ||
*/ | ||
inline fun <reified A : Enum<A>> enumFromStringSafe(value: String) : A? { | ||
return enumValues<A>().find { s -> s.name == value.uppercase() } | ||
inline fun <reified A : Enum<A>> enumSetFromInput(commaSeparatedString: String) : Set<A> { | ||
return commaSeparatedString.split(',').filter(String::isNotBlank).map { enumValueOf<A>(it) }.toSet() | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,28 @@ | ||
package com.gmtkgamejam.koin | ||
|
||
import com.gmtkgamejam.Config | ||
import org.apache.hc.core5.http.HttpHost | ||
import org.koin.dsl.module | ||
import org.opensearch.client.json.jackson.JacksonJsonpMapper | ||
import org.opensearch.client.opensearch.OpenSearchClient | ||
import org.opensearch.client.transport.OpenSearchTransport | ||
import org.opensearch.client.transport.httpclient5.ApacheHttpClient5TransportBuilder | ||
|
||
val SearchEngineModule = module(createdAtStart = true) { | ||
single<OpenSearchClient> { | ||
// Depending on hosting requirements, this is likely `http` while inside a VPC | ||
val httpHost = HttpHost( | ||
// TODO: Consider if there's a more flexible connection option w/o going into cert management hell | ||
Config.getString("se.protocol"), | ||
Config.getString("se.host"), | ||
Config.getInt("se.port") | ||
) | ||
|
||
val transport: OpenSearchTransport = ApacheHttpClient5TransportBuilder | ||
.builder(httpHost) | ||
.setMapper(JacksonJsonpMapper()) | ||
.build() | ||
|
||
OpenSearchClient(transport) | ||
} | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,34 @@ | ||
package com.gmtkgamejam.models.posts | ||
|
||
/** | ||
* Data model for an entry in the search engine | ||
* | ||
* This is a stripped-down PostItem with just the fields we use for searching | ||
* | ||
* Any index mappings (i.e. the `description_shingle` defined in OpensearchClusterConfigurer) will be applied and | ||
* computed by the SE at index-time | ||
*/ | ||
data class SearchItem( | ||
val id: String, | ||
var description_shingle: String, | ||
var size: Int, | ||
var skillsPossessed: Set<Skills>?, | ||
var skillsSought: Set<Skills>?, | ||
var preferredTools: Set<Tools>?, | ||
var availability: Availability, | ||
var timezoneOffsets: Set<Int>, | ||
var languages: Set<String>, | ||
) { | ||
constructor(postItem: PostItem): this( | ||
id = postItem.id, | ||
// TODO: what work do we want to do here that the SE won't already do? | ||
description_shingle = postItem.description.lowercase().replace("\n", " "), | ||
size = postItem.size, | ||
skillsPossessed = postItem.skillsPossessed, | ||
skillsSought = postItem.skillsSought, | ||
preferredTools = postItem.preferredTools, | ||
availability = postItem.availability, | ||
timezoneOffsets = postItem.timezoneOffsets, | ||
languages = postItem.languages, | ||
) | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,28 @@ | ||
package com.gmtkgamejam.routing | ||
|
||
import com.gmtkgamejam.models.posts.PostItem | ||
import com.gmtkgamejam.respondJSON | ||
import com.gmtkgamejam.search.OpensearchClusterConfigurer | ||
import com.gmtkgamejam.services.PostService | ||
import io.ktor.http.* | ||
import io.ktor.server.application.* | ||
import io.ktor.server.routing.* | ||
import org.litote.kmongo.eq | ||
|
||
// TODO: Auth control | ||
fun Application.configureInfraRouting() { | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. The This route will load all posts from the DB and set up the This routing block will likely use a secret configuration value (set as a runtime environment variable) as access control. |
||
|
||
val postService = PostService() | ||
|
||
routing { | ||
route("/infra") { | ||
route("/se") { | ||
get("/reset") { | ||
val posts = postService.getPosts(PostItem::deletedAt eq null) | ||
OpensearchClusterConfigurer().initCluster(posts) | ||
call.respondJSON("Search engine reset complete", HttpStatusCode.OK) | ||
} | ||
} | ||
} | ||
} | ||
} |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
The
httpcore
libraries provide HTTP request support for the Opensearch client, which makes requests over HTTP.There are a range of Kotlin-specific Opensearch clients; I've not yet found one that wasn't a complete pig to set up or wasn't completely baffling to wrap my head around. I'm certain they exist, but didn't have any luck on a quick first pass.
Although the Java client is extremely verbose I've used it for this first version as it's very concise to form the initial connection to Opensearch, and constructing the query can be tidied up with helper methods if we want to go down that road.