Failure modes, and the tests that catch them

1
    testImplementation("org.springframework.boot:spring-boot-starter-test")
2
    testImplementation("org.springframework.boot:spring-boot-testcontainers")
3
    testImplementation("org.springframework.security:spring-security-test")
4
    testImplementation("org.testcontainers:testcontainers-elasticsearch")
5
    testImplementation("org.testcontainers:testcontainers-junit-jupiter")
6
    testImplementation("org.jetbrains.kotlin:kotlin-test-junit5")
7
    testRuntimeOnly("org.junit.platform:junit-platform-launcher")
8
}
9

10
tasks.test {
11
    useJUnitPlatform()
12
}

1
package `in`.o612.eng.usersearch.api.search
2

3
import co.elastic.clients.elasticsearch.core.SearchRequest
4
import co.elastic.clients.json.JsonpUtils
5
import `in`.o612.eng.usersearch.index.ElasticsearchJson
6
import tools.jackson.databind.JsonNode
7
import tools.jackson.databind.json.JsonMapper
8

9
/** The JSON body a request would send, as a tree that tests can navigate. */
10
fun SearchRequest.toJsonTree(): JsonNode =
11
    JsonMapper.builder().build().readTree(JsonpUtils.toJsonString(this, ElasticsearchJson.jsonpMapper()))
12

13
/** The elements of a JSON array node. */
14
fun JsonNode.items(): List<JsonNode> = (0 until size()).map { get(it) }

1
package `in`.o612.eng.usersearch.api.search
2

3
import `in`.o612.eng.usersearch.api.web.NameMode
4
import `in`.o612.eng.usersearch.api.web.SortOrder
5
import `in`.o612.eng.usersearch.api.web.UserSearchRequest
6
import org.junit.jupiter.api.Test
7
import kotlin.test.assertEquals
8
import kotlin.test.assertTrue
9

10
class UserQueryBuilderTest {
11

12
    @Test
13
    fun `exact constraints are filters and only the name clause is scored`() {
14
        val json = UserQueryBuilder.build(UserSearchRequest(name = "prashant", state = "bihar", city = "patna")).toJsonTree()
15
        val bool = json.at("/query/bool")
16

17
        assertEquals(1, bool.at("/must").size(), "one scored clause")
18
        val filterFields = bool.at("/filter").items().map { it.properties().first().value.properties().first().key }
19
        assertEquals(listOf("accountStatus", "state", "city"), filterFields)
20
    }
21

22
    @Test
23
    fun `cross_fields forces one analyser so words may match in different fields`() {
24
        val json = UserQueryBuilder.build(UserSearchRequest(name = "prashant patna")).toJsonTree()
25
        val crossFields = json.at("/query/bool/must/0/bool/should/0/multi_match")
26

27
        assertEquals("cross_fields", crossFields.at("/type").asString())
28
        assertEquals("name_search", crossFields.at("/analyzer").asString())
29
        assertEquals("1", json.at("/query/bool/must/0/bool/minimum_should_match").asString())
30
    }
31

32
    @Test
33
    fun `every sort ends with the unique userId`() {
34
        for (order in SortOrder.entries) {
35
            val sort = UserQueryBuilder.build(UserSearchRequest(sort = order)).toJsonTree().at("/sort")
36
            assertEquals("userId", sort.items().last().properties().first().key, "sort $order must be total")
37
        }
38
    }
39

40
    @Test
41
    fun `prefix mode searches the edge n-gram field without fuzziness`() {
42
        val json = UserQueryBuilder.build(UserSearchRequest(name = "pras", nameMode = NameMode.PREFIX)).toJsonTree()
43
        val match = json.at("/query/bool/must/0/match/fullName.prefix")
44

45
        assertEquals("and", match.at("/operator").asString())
46
        assertTrue(match.at("/fuzziness").isMissingNode)
47
    }
48

49
    @Test
50
    fun `a point-in-time page names no index and continues after the cursor`() {
51
        val request = UserQueryBuilder.build(
52
            UserSearchRequest(city = "patna", sort = SortOrder.UPDATED_AT),
53
            pitId = "pit-1",
54
            searchAfter = SearchCursor(listOf(1787702400000L, "385496"), null, "x").searchAfter(),
55
        )
56

57
        assertTrue(request.index().isEmpty(), "a PIT search must not name an index")
58
        assertEquals("pit-1", request.pit()!!.id())
59
        assertEquals(2, request.searchAfter().size)
60
    }
61

62
    @Test
63
    fun `the caller's scope restricts searches and every facet`() {
64
        val scope = SearchScope(setOf("bihar"))
65
        val request = UserSearchRequest(name = "prashant")
66

67
        val searchFilters = UserQueryBuilder.build(request, scope).toJsonTree().at("/query/bool/filter")
68
        assertTrue(searchFilters.items().any { it.at("/terms/state").isArray }, "search must carry the scope")
69

70
        // Regression test for the facet leak in chapter 16: the scope must be in the shared base query,
71
        // which every facet applies, not in the state facet's own filter, which that facet ignores.
72
        val facetBase = UserFacetsBuilder.build(request, scope).toJsonTree().at("/query/bool/filter")
73
        assertTrue(facetBase.items().any { it.at("/terms/state").isArray }, "facets must carry the scope")
74
    }
75

76
    @Test
77
    fun `a facet ignores its own filter and applies the others`() {
78
        val json = UserFacetsBuilder.build(UserSearchRequest(state = "bihar", city = "patna")).toJsonTree()
79

80
        val cityFacetFilters = json.at("/aggregations/city/filter/bool/filter").items().map { it.at("/term").properties().first().key }
81
        val stateFacetFilters = json.at("/aggregations/state/filter/bool/filter").items().map { it.at("/term").properties().first().key }
82
        assertEquals(listOf("state"), cityFacetFilters)
83
        assertEquals(listOf("city"), stateFacetFilters)
84
    }
85
}

1
package `in`.o612.eng.usersearch.api.search
2

3
import co.elastic.clients.elasticsearch._types.FieldValue
4
import `in`.o612.eng.usersearch.api.web.UserSearchRequest
5
import org.junit.jupiter.api.Test
6
import org.junit.jupiter.api.assertThrows
7
import kotlin.test.assertEquals
8

9
class SearchCursorTest {
10

11
    private val request = UserSearchRequest(city = "patna")
12

13
    @Test
14
    fun `a cursor round-trips its sort values with their types`() {
15
        val sortValues = listOf(FieldValue.of(25.586945), FieldValue.of(1761696000000L), FieldValue.of("343200"))
16
        val decoded = SearchCursor.decode(SearchCursor.from(sortValues, null, request).encode(), request)
17

18
        assertEquals(listOf(25.586945, 1761696000000L, "343200"), decoded.searchAfter().map { it._get() })
19
    }
20

21
    @Test
22
    fun `a cursor is rejected for a different search, but not for a different page size`() {
23
        val token = SearchCursor.from(listOf(FieldValue.of("1")), null, request).encode()
24

25
        SearchCursor.decode(token, request.copy(size = 50))
26
        assertThrows<InvalidCursorException> { SearchCursor.decode(token, request.copy(city = "gaya")) }
27
        assertThrows<InvalidCursorException> { SearchCursor.decode("not-a-cursor", request) }
28
    }
29
}

1
package `in`.o612.eng.usersearch.api.it
2

3
import org.springframework.boot.testcontainers.service.connection.ServiceConnection
4
import org.springframework.boot.testcontainers.service.connection.Ssl
5
import org.testcontainers.elasticsearch.ElasticsearchContainer
6

7
/**
8
 * One Elasticsearch for every integration test class, with security and TLS on, as in production.
9
 * Spring Boot takes the URL, the elastic user's password, and the container's CA from it.
10
 */
11
object SharedElasticsearch {
12
    @JvmField
13
    @ServiceConnection
14
    @Ssl // without it, Spring Boot connects over plain HTTP and the TLS-only container closes the connection
15
    val elasticsearch: ElasticsearchContainer =
16
        ElasticsearchContainer("docker.elastic.co/elasticsearch/elasticsearch:9.5.4")
17
            .withEnv("ES_JAVA_OPTS", "-Xms512m -Xmx512m")
18
            // Started here, not by Spring: the CA certificate is read while test properties are resolved.
19
            .apply { start() }
20
}

1
package `in`.o612.eng.usersearch.api.it
2

3
import co.elastic.clients.elasticsearch.ElasticsearchClient
4
import co.elastic.clients.elasticsearch._types.Refresh
5
import `in`.o612.eng.usersearch.index.UserProfileIndex
6
import org.springframework.test.context.DynamicPropertyRegistry
7
import java.nio.file.Files
8

9
object IntegrationTestSupport {
10
    /** Properties the application normally reads from the environment, set for the test container. */
11
    fun register(registry: DynamicPropertyRegistry) {
12
        val ca = Files.createTempFile("es-test-ca", ".crt")
13
        Files.write(ca, SharedElasticsearch.elasticsearch.caCertAsBytes().orElseThrow())
14
        registry.add("ELASTICSEARCH_CA_CERT") { "file:$ca" }
15
        registry.add("ELASTICSEARCH_URIS") { "https://localhost" } // replaced by the service connection
16
        registry.add("ELASTICSEARCH_API_KEY") { "" }                 // the service connection uses the elastic user
17
        registry.add("USER_SEARCH_INDEX_BOOTSTRAP") { "true" }
18
        registry.add("LAB_AGENT_PASSWORD") { AGENT_PASSWORD }
19
        registry.add("LAB_SUPERVISOR_PASSWORD") { SUPERVISOR_PASSWORD }
20
    }
21

22
    const val AGENT_PASSWORD = "agent-test"
23
    const val SUPERVISOR_PASSWORD = "supervisor-test"
24
    const val PROFILES = 400
25

26
    @Volatile
27
    private var seeded = false
28

29
    /** Indexes the synthetic profiles once per JVM, and waits until they are searchable. */
30
    @Synchronized
31
    fun seed(client: ElasticsearchClient) {
32
        if (seeded) return
33
        val response = client.bulk { b ->
34
            b.refresh(Refresh.WaitFor)
35
            TestProfiles.generate(PROFILES).forEach { doc ->
36
                b.operations { op -> op.index { i -> i.index(UserProfileIndex.WRITE_ALIAS).id(doc.userId).document(doc) } }
37
            }
38
            b
39
        }
40
        check(!response.errors()) { "Seeding failed: ${response.items().firstOrNull { it.error() != null }?.error()}" }
41
        seeded = true
42
    }
43
}

1
package `in`.o612.eng.usersearch.api.it
2

3
import `in`.o612.eng.usersearch.index.UserProfileDocument
4
import java.time.Instant
5
import java.time.LocalDate
6
import java.time.temporal.ChronoUnit
7

8
/**
9
 * Deterministic synthetic profiles. Nothing is copied from real data: names, places, and identifiers
10
 * are generated, with enough variety to exercise analysis, filters, and paging.
11
 */
12
object TestProfiles {
13
    private val firstNames = listOf("Prashant", "Priya", "Mohammed", "José", "Lakshmi", "Rahul", "Fatima", "Arjun")
14
    private val lastNames = listOf("Kumar", "Sharma", "Khan", "Fernandes", "Iyer", "Jha")
15
    private val places = listOf("Patna" to "Bihar", "Gaya" to "Bihar", "Kochi" to "Kerala", "Mumbai" to "Maharashtra")
16
    private val statuses = listOf("ACTIVE", "ACTIVE", "ACTIVE", "INACTIVE")
17
    private val start: Instant = Instant.parse("2026-01-01T00:00:00Z")
18

19
    fun generate(count: Int): List<UserProfileDocument> = (1..count).map { id ->
20
        val first = firstNames[id % firstNames.size]
21
        val last = lastNames[(id / firstNames.size) % lastNames.size]
22
        val (city, state) = places[(id * 7) % places.size]
23
        UserProfileDocument(
24
            userId = id.toString(),
25
            fullName = "$first $last",
26
            firstName = first,
27
            lastName = last,
28
            email = "${first.lowercase()}.${last.lowercase()}.$id@example.com",
29
            mobileNumber = "9" + id.toString().padStart(9, '0'),
30
            city = city,
31
            state = state,
32
            country = "IN",
33
            pincode = "800" + (id % 1000).toString().padStart(3, '0'),
34
            dateOfBirth = LocalDate.of(1970, 1, 1).plusDays(id * 37L % 15000),
35
            gender = if (id % 2 == 0) "FEMALE" else "MALE",
36
            accountStatus = statuses[id % statuses.size],
37
            createdAt = start,
38
            updatedAt = start.plus(id.toLong(), ChronoUnit.HOURS),
39
        )
40
    }
41
}

1
package `in`.o612.eng.usersearch.api.index
2

3
import co.elastic.clients.elasticsearch.ElasticsearchClient
4
import co.elastic.clients.elasticsearch._types.ElasticsearchException
5
import co.elastic.clients.elasticsearch._types.HealthStatus
6
import co.elastic.clients.transport.TransportException
7
import `in`.o612.eng.usersearch.index.IndexAdmin
8
import `in`.o612.eng.usersearch.index.UserProfileIndex
9
import org.slf4j.LoggerFactory
10
import org.springframework.boot.ApplicationArguments
11
import org.springframework.boot.ApplicationRunner
12
import org.springframework.boot.autoconfigure.condition.ConditionalOnBooleanProperty
13
import org.springframework.stereotype.Component
14
import java.time.Duration
15
import java.time.Instant
16

17
/**
18
 * Creates the current index version and its aliases when none exist.
19
 * Enabled only with user-search.index.bootstrap=true, for local development and tests.
20
 */
21
@Component
22
@ConditionalOnBooleanProperty("user-search.index.bootstrap")
23
class IndexBootstrapper(
24
    private val indexAdmin: IndexAdmin,
25
    private val client: ElasticsearchClient,
26
) : ApplicationRunner {
27

28
    private val log = LoggerFactory.getLogger(javaClass)
29

30
    override fun run(args: ApplicationArguments) {
31
        awaitCluster(Duration.ofSeconds(60))
32
        val targets = indexAdmin.aliasTargets(UserProfileIndex.READ_ALIAS)
33
        if (targets.isNotEmpty()) {
34
            log.info("{} already points to {}; nothing to create", UserProfileIndex.READ_ALIAS, targets)
35
            return
36
        }
37
        // The current version, not v1: the code depends on its fields, such as the facets' display sub-fields.
38
        val index = UserProfileIndex.indexName(UserProfileIndex.CURRENT_VERSION)
39
        val created = indexAdmin.createIndex(UserProfileIndex.CURRENT_VERSION)
40
        if (indexAdmin.aliasTargets(UserProfileIndex.READ_ALIAS).isEmpty()) indexAdmin.addAliases(index)
41
        log.info("Index {} created: {}", index, created)
42
    }
43

44
    /**
45
     * A node that has just started answers 503 while it recovers its cluster state, even to a health
46
     * request that asks to wait (chapter 17). Retry until the cluster is at least yellow, or give up.
47
     */
48
    private fun awaitCluster(limit: Duration) {
49
        val deadline = Instant.now().plus(limit)
50
        while (true) {
51
            try {
52
                val health = client.cluster().health { it.waitForStatus(HealthStatus.Yellow).timeout { t -> t.time("10s") } }
53
                if (!health.timedOut()) return
54
            } catch (e: ElasticsearchException) {
55
                log.info("Cluster not ready yet: {}", e.status())
56
            } catch (e: TransportException) {
57
                log.info("Cluster not ready yet: {}", e.message)
58
            }
59
            check(Instant.now().isBefore(deadline)) { "Elasticsearch was not ready within $limit" }
60
            Thread.sleep(1_000)
61
        }
62
    }
63
}

1
    /** The index version the applications' code is written for. */
2
    const val CURRENT_VERSION = 2

1
    /**
2
     * Creates an index named [name], `user-profile-v<version>` by default, from the reviewed definition of
3
     * [version], including any aliases the definition declares. Returns false, and changes nothing, if it exists.
4
     */
5
    fun createIndex(version: Int, name: String = UserProfileIndex.indexName(version)): Boolean {
6
        if (client.indices().exists { it.index(name) }.value()) return false

1
    /** Points both aliases at [index], for a cluster that has none yet. */
2
    fun addAliases(index: String) {
3
        client.indices().updateAliases { u ->
4
            u.actions { a -> a.add { ad -> ad.index(index).alias(UserProfileIndex.READ_ALIAS) } }
5
                .actions { a -> a.add { ad -> ad.index(index).alias(UserProfileIndex.WRITE_ALIAS).isWriteIndex(true) } }
6
        }
7
    }

1
package `in`.o612.eng.usersearch.api.it
2

3
import co.elastic.clients.elasticsearch.ElasticsearchClient
4
import `in`.o612.eng.usersearch.api.search.UserSearchService
5
import `in`.o612.eng.usersearch.api.web.FacetsResponse
6
import `in`.o612.eng.usersearch.api.web.NameMode
7
import `in`.o612.eng.usersearch.api.web.SortOrder
8
import `in`.o612.eng.usersearch.api.web.UserSearchRequest
9
import `in`.o612.eng.usersearch.index.IndexAdmin
10
import `in`.o612.eng.usersearch.index.UserProfileIndex
11
import org.junit.jupiter.api.BeforeEach
12
import org.junit.jupiter.api.Test
13
import org.springframework.beans.factory.annotation.Autowired
14
import org.springframework.beans.factory.annotation.Value
15
import org.springframework.boot.test.context.SpringBootTest
16
import org.springframework.boot.testcontainers.context.ImportTestcontainers
17
import org.springframework.test.context.DynamicPropertyRegistry
18
import org.springframework.test.context.DynamicPropertySource
19
import org.springframework.web.client.RestClient
20
import kotlin.test.assertEquals
21
import kotlin.test.assertTrue
22

23
@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
24
@ImportTestcontainers(SharedElasticsearch::class)
25
class UserSearchIntegrationTest {
26

27
    @Autowired lateinit var client: ElasticsearchClient
28
    @Autowired lateinit var indexAdmin: IndexAdmin
29
    @Autowired lateinit var service: UserSearchService
30
    @Value("\${local.server.port}") var port: Int = 0
31

32
    @BeforeEach
33
    fun seed() = IntegrationTestSupport.seed(client)
34

35
    @Test
36
    fun `the bootstrapper creates the current version, strict, behind both aliases`() {
37
        val current = UserProfileIndex.indexName(UserProfileIndex.CURRENT_VERSION)
38
        assertEquals(setOf(current), indexAdmin.aliasTargets(UserProfileIndex.READ_ALIAS))
39
        assertEquals(setOf(current), indexAdmin.aliasTargets(UserProfileIndex.WRITE_ALIAS))
40
        val mapping = client.indices().getMapping { it.index(current) }[current]!!.mappings()
41
        assertEquals("strict", mapping.dynamic()!!.jsonValue())
42
    }
43

44
    @Test
45
    fun `autocomplete, typos, synonyms, and filters find the expected profiles`() {
46
        val all = TestProfiles.generate(IntegrationTestSupport.PROFILES).filter { it.accountStatus == "ACTIVE" }
47

48
        fun count(request: UserSearchRequest) = service.search(request).total.value
49

50
        assertEquals(all.count { it.firstName == "Prashant" }.toLong(), count(UserSearchRequest(name = "pras", nameMode = NameMode.PREFIX)))
51
        assertEquals(all.count { it.fullName == "Prashant Kumar" }.toLong(), count(UserSearchRequest(name = "prashnat kumr")))
52
        assertEquals(all.count { it.firstName == "Mohammed" }.toLong(), count(UserSearchRequest(name = "mohd")))
53
        assertEquals(all.count { it.firstName == "José" }.toLong(), count(UserSearchRequest(name = "JOSE")))
54
        assertEquals(all.count { it.city == "Patna" }.toLong(), count(UserSearchRequest(city = "PATNA")))
55
    }
56

57
    @Test
58
    fun `a stable walk returns every match exactly once`() {
59
        val expected = TestProfiles.generate(IntegrationTestSupport.PROFILES).count { it.accountStatus == "ACTIVE" && it.state == "Bihar" }
60
        val seen = mutableListOf<String>()
61
        var request = UserSearchRequest(state = "bihar", sort = SortOrder.UPDATED_AT, size = 7, stable = true)
62
        while (true) {
63
            val page = service.search(request)
64
            seen += page.hits.map { it.userId }
65
            request = request.copy(cursor = page.nextCursor ?: break)
66
        }
67
        assertEquals(expected, seen.size)
68
        assertEquals(seen.size, seen.toSet().size, "no profile twice")
69
    }
70

71
    @Test
72
    fun `an agent's facets never include states outside their scope`() {
73
        val rest = RestClient.create("http://localhost:$port")
74
        val facets = rest.get().uri("/api/users/facets")
75
            .headers { it.setBasicAuth("agent", IntegrationTestSupport.AGENT_PASSWORD) }
76
            .retrieve().body(FacetsResponse::class.java)!!
77
        assertEquals(listOf("Bihar"), facets.facets.getValue("state").map { it.value })
78

79
        val supervisorFacets = rest.get().uri("/api/users/facets")
80
            .headers { it.setBasicAuth("supervisor", IntegrationTestSupport.SUPERVISOR_PASSWORD) }
81
            .retrieve().body(FacetsResponse::class.java)!!
82
        assertTrue(supervisorFacets.facets.getValue("state").size > 1)
83
    }
84

85
    companion object {
86
        @JvmStatic
87
        @DynamicPropertySource
88
        fun properties(registry: DynamicPropertyRegistry) = IntegrationTestSupport.register(registry)
89
    }
90
}

1
package `in`.o612.eng.usersearch.api.it
2

3
import co.elastic.clients.elasticsearch.ElasticsearchClient
4
import co.elastic.clients.elasticsearch._types.Refresh
5
import `in`.o612.eng.usersearch.index.IndexAdmin
6
import `in`.o612.eng.usersearch.index.UserProfileIndex
7
import org.junit.jupiter.api.BeforeEach
8
import org.junit.jupiter.api.Test
9
import org.springframework.beans.factory.annotation.Autowired
10
import org.springframework.boot.test.context.SpringBootTest
11
import org.springframework.boot.testcontainers.context.ImportTestcontainers
12
import org.springframework.test.context.DynamicPropertyRegistry
13
import org.springframework.test.context.DynamicPropertySource
14
import kotlin.test.assertEquals
15

16
@SpringBootTest
17
@ImportTestcontainers(SharedElasticsearch::class)
18
class AliasSwapIntegrationTest {
19

20
    @Autowired lateinit var client: ElasticsearchClient
21
    @Autowired lateinit var indexAdmin: IndexAdmin
22

23
    @BeforeEach
24
    fun seed() = IntegrationTestSupport.seed(client)
25

26
    @Test
27
    fun `swapping aliases moves reads and writes to the new index, and back`() {
28
        val current = UserProfileIndex.indexName(UserProfileIndex.CURRENT_VERSION)
29
        val candidate = "$current-candidate"
30
        indexAdmin.createIndex(UserProfileIndex.CURRENT_VERSION, name = candidate)
31
        // The candidate holds one profile that the current index does not.
32
        val newcomer = TestProfiles.generate(IntegrationTestSupport.PROFILES + 1).last()
33
        client.index { it.index(candidate).id(newcomer.userId).document(newcomer).refresh(Refresh.WaitFor) }
34

35
        try {
36
            indexAdmin.swapAliases(fromIndex = current, toIndex = candidate)
37
            assertEquals(setOf(candidate), indexAdmin.aliasTargets(UserProfileIndex.READ_ALIAS))
38
            assertEquals(setOf(candidate), indexAdmin.aliasTargets(UserProfileIndex.WRITE_ALIAS))
39
            assertEquals(1L, client.count { it.index(UserProfileIndex.READ_ALIAS) }.count())
40
        } finally {
41
            indexAdmin.swapAliases(fromIndex = candidate, toIndex = current) // the rollback path
42
            client.indices().delete { it.index(candidate) }
43
        }
44
        assertEquals(setOf(current), indexAdmin.aliasTargets(UserProfileIndex.READ_ALIAS))
45
        assertEquals(IntegrationTestSupport.PROFILES.toLong(), client.count { it.index(UserProfileIndex.READ_ALIAS) }.count())
46
    }
47

48
    companion object {
49
        @JvmStatic
50
        @DynamicPropertySource
51
        fun properties(registry: DynamicPropertyRegistry) = IntegrationTestSupport.register(registry)
52
    }
53
}