-
Notifications
You must be signed in to change notification settings - Fork 7
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Use Opensearch as the tool for searching posts
Full details and diagrams will be in the PR (#296), but the core details are: * the previous searching tool was extremely naive, entirely using `String.contains` * there was no spell correction, nor easy way to add it in * results came back in an arbitrary order and weren't easily scored A search engine ('SE' for short, used in the codebase) is a much more appropriate tool. This change migrates the searching, scoring, and ranking logic to Opensearch to return a list of ordered SearchItem instances, which we then use to return a list of ordered PostItem instances to the user.
- Loading branch information
1 parent
5d1a653
commit f2a81b9
Showing
23 changed files
with
510 additions
and
174 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
21 changes: 21 additions & 0 deletions
21
api/src/main/kotlin/com/gmtkgamejam/ApplicationCallExtensions.kt
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,21 @@ | ||
package com.gmtkgamejam | ||
|
||
import com.auth0.jwt.JWT | ||
import com.gmtkgamejam.models.auth.AuthTokenSet | ||
import com.gmtkgamejam.services.AuthService | ||
import io.ktor.http.* | ||
import io.ktor.server.application.* | ||
import io.ktor.server.request.* | ||
import io.ktor.server.response.* | ||
|
||
fun ApplicationCall.getAuthTokenSet(authService: AuthService): AuthTokenSet? { | ||
return this.request.header("Authorization") | ||
?.substringAfter("Bearer ") | ||
?.let { JWT.decode(it) }?.getClaim("id")?.asString() | ||
?.let { authService.getTokenSet(it) } | ||
} | ||
|
||
suspend fun ApplicationCall.respondJSON(text: String, status: HttpStatusCode? = null) { | ||
status?.let { response.status(it) } | ||
respond(mapOf("message" to text)) | ||
} |
16 changes: 0 additions & 16 deletions
16
api/src/main/kotlin/com/gmtkgamejam/ApplicationResponseExtensions.kt
This file was deleted.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,10 +1,5 @@ | ||
package com.gmtkgamejam | ||
|
||
/** | ||
* Floating function to cast a String to an Enum without throwing an exception | ||
* | ||
* Suggest using with mapNotNull{} where possible | ||
*/ | ||
inline fun <reified A : Enum<A>> enumFromStringSafe(value: String) : A? { | ||
return enumValues<A>().find { s -> s.name == value.uppercase() } | ||
inline fun <reified A : Enum<A>> enumSetFromInput(commaSeparatedString: String) : Set<A> { | ||
return commaSeparatedString.split(',').filter(String::isNotBlank).map { enumValueOf<A>(it) }.toSet() | ||
} |
28 changes: 28 additions & 0 deletions
28
api/src/main/kotlin/com/gmtkgamejam/koin/SearchEngineModule.kt
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,28 @@ | ||
package com.gmtkgamejam.koin | ||
|
||
import com.gmtkgamejam.Config | ||
import org.apache.hc.core5.http.HttpHost | ||
import org.koin.dsl.module | ||
import org.opensearch.client.json.jackson.JacksonJsonpMapper | ||
import org.opensearch.client.opensearch.OpenSearchClient | ||
import org.opensearch.client.transport.OpenSearchTransport | ||
import org.opensearch.client.transport.httpclient5.ApacheHttpClient5TransportBuilder | ||
|
||
val SearchEngineModule = module(createdAtStart = true) { | ||
single<OpenSearchClient> { | ||
// Depending on hosting requirements, this is likely `http` while inside a VPC | ||
val httpHost = HttpHost( | ||
// TODO: Consider if there's a more flexible connection option w/o going into cert management hell | ||
Config.getString("se.protocol"), | ||
Config.getString("se.host"), | ||
Config.getInt("se.port") | ||
) | ||
|
||
val transport: OpenSearchTransport = ApacheHttpClient5TransportBuilder | ||
.builder(httpHost) | ||
.setMapper(JacksonJsonpMapper()) | ||
.build() | ||
|
||
OpenSearchClient(transport) | ||
} | ||
} |
34 changes: 34 additions & 0 deletions
34
api/src/main/kotlin/com/gmtkgamejam/models/posts/SearchItem.kt
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,34 @@ | ||
package com.gmtkgamejam.models.posts | ||
|
||
/** | ||
* Data model for an entry in the search engine | ||
* | ||
* This is a stripped-down PostItem with just the fields we use for searching | ||
* | ||
* Any index mappings (i.e. the `description_shingle` defined in OpensearchClusterConfigurer) will be applied and | ||
* computed by the SE at index-time | ||
*/ | ||
data class SearchItem( | ||
val id: String, | ||
var description_shingle: String, | ||
var size: Int, | ||
var skillsPossessed: Set<Skills>?, | ||
var skillsSought: Set<Skills>?, | ||
var preferredTools: Set<Tools>?, | ||
var availability: Availability, | ||
var timezoneOffsets: Set<Int>, | ||
var languages: Set<String>, | ||
) { | ||
constructor(postItem: PostItem): this( | ||
id = postItem.id, | ||
// TODO: what work do we want to do here that the SE won't already do? | ||
description_shingle = postItem.description.lowercase().replace("\n", " "), | ||
size = postItem.size, | ||
skillsPossessed = postItem.skillsPossessed, | ||
skillsSought = postItem.skillsSought, | ||
preferredTools = postItem.preferredTools, | ||
availability = postItem.availability, | ||
timezoneOffsets = postItem.timezoneOffsets, | ||
languages = postItem.languages, | ||
) | ||
} |
28 changes: 28 additions & 0 deletions
28
api/src/main/kotlin/com/gmtkgamejam/routing/InfraRoutes.kt
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,28 @@ | ||
package com.gmtkgamejam.routing | ||
|
||
import com.gmtkgamejam.models.posts.PostItem | ||
import com.gmtkgamejam.respondJSON | ||
import com.gmtkgamejam.search.OpensearchClusterConfigurer | ||
import com.gmtkgamejam.services.PostService | ||
import io.ktor.http.* | ||
import io.ktor.server.application.* | ||
import io.ktor.server.routing.* | ||
import org.litote.kmongo.eq | ||
|
||
// TODO: Auth control | ||
fun Application.configureInfraRouting() { | ||
|
||
val postService = PostService() | ||
|
||
routing { | ||
route("/infra") { | ||
route("/se") { | ||
get("/reset") { | ||
val posts = postService.getPosts(PostItem::deletedAt eq null) | ||
OpensearchClusterConfigurer().initCluster(posts) | ||
call.respondJSON("Search engine reset complete", HttpStatusCode.OK) | ||
} | ||
} | ||
} | ||
} | ||
} |
Oops, something went wrong.