diff --git a/backend/apps/dev/build.gradle b/backend/apps/dev/build.gradle index ced247e784..29475cd733 100644 --- a/backend/apps/dev/build.gradle +++ b/backend/apps/dev/build.gradle @@ -51,6 +51,9 @@ dependencies { implementation "org.springframework.boot:spring-boot-starter-webflux" implementation "org.springframework.boot:spring-boot-starter-security" + // OpenSearch client for InsecureSslOpenSearchClientConfig + implementation "org.opensearch.client:spring-data-opensearch-starter:${springDataOpenSearchVersion}" + // Spring cloud stream RabbitMQ implementation "org.springframework.cloud:spring-cloud-starter-stream-rabbit:${springCloudStreamVersion}" implementation "com.rabbitmq:amqp-client:$amqpCLientVersion" diff --git a/backend/apps/dev/docker-compose.yaml b/backend/apps/dev/docker-compose.yaml index a7be480dce..afaa1a1af0 100644 --- a/backend/apps/dev/docker-compose.yaml +++ b/backend/apps/dev/docker-compose.yaml @@ -82,6 +82,24 @@ services: volumes: - gzac-database-data-mysql:/var/lib/mysql # persist data even if container shuts down + gzac-opensearch: + container_name: gzac-docker-compose-gzac-opensearch + image: opensearchproject/opensearch:2.19.2 + ports: + - "9200:9200" + environment: + - discovery.type=single-node + - OPENSEARCH_INITIAL_ADMIN_PASSWORD=2cr8zkOoEXQJ3xUk!Aa1 + - OPENSEARCH_JAVA_OPTS=-Xms256m -Xmx256m + volumes: + - gzac-opensearch-data:/usr/share/opensearch/data + healthcheck: + test: [ "CMD-SHELL", "curl -sf -u admin:2cr8zkOoEXQJ3xUk!Aa1 https://localhost:9200/_cluster/health -k || exit 1" ] + interval: 10s + timeout: 5s + retries: 12 + start_period: 30s + gzac-rabbitmq: image: rabbitmq:4.1.0-management container_name: gzac-docker-compose-gzac-rabbitmq @@ -733,3 +751,4 @@ services: volumes: gzac-database-data: gzac-database-data-mysql: + gzac-opensearch-data: diff --git a/backend/apps/dev/src/main/kotlin/com/ritense/gzac/opensearch/InsecureSslOpenSearchClientConfig.kt b/backend/apps/dev/src/main/kotlin/com/ritense/gzac/opensearch/InsecureSslOpenSearchClientConfig.kt new file mode 100644 index 0000000000..7b99905f44 --- /dev/null +++ b/backend/apps/dev/src/main/kotlin/com/ritense/gzac/opensearch/InsecureSslOpenSearchClientConfig.kt @@ -0,0 +1,74 @@ +/* + * Copyright 2015-2026 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.ritense.gzac.opensearch + +import org.apache.http.auth.AuthScope +import org.apache.http.auth.UsernamePasswordCredentials +import org.apache.http.conn.ssl.NoopHostnameVerifier +import org.apache.http.conn.ssl.TrustAllStrategy +import org.apache.http.impl.client.BasicCredentialsProvider +import org.apache.http.ssl.SSLContextBuilder +import org.opensearch.client.RestClientBuilder +import org.opensearch.spring.boot.autoconfigure.RestClientBuilderCustomizer +import org.springframework.beans.factory.annotation.Value +import org.springframework.context.annotation.Bean +import org.springframework.context.annotation.Configuration + +/** + * Dev-only configuration that disables SSL certificate verification for OpenSearch + * and configures basic authentication. + * Allows connecting to OpenSearch with self-signed certificates in local development. + * + * DO NOT use this configuration in production. + */ +@Configuration +class InsecureSslOpenSearchClientConfig { + + @Value("\${opensearch.username:}") + private lateinit var username: String + + @Value("\${opensearch.password:}") + private lateinit var password: String + + @Bean + fun insecureSslRestClientBuilderCustomizer(): RestClientBuilderCustomizer { + return object : RestClientBuilderCustomizer { + override fun customize(builder: RestClientBuilder) { + builder.setHttpClientConfigCallback { httpClientBuilder -> + val sslContext = SSLContextBuilder.create() + .loadTrustMaterial(TrustAllStrategy.INSTANCE) + .build() + + httpClientBuilder + .setSSLContext(sslContext) + .setSSLHostnameVerifier(NoopHostnameVerifier.INSTANCE) + + if (username.isNotBlank() && password.isNotBlank()) { + val credentialsProvider = BasicCredentialsProvider() + credentialsProvider.setCredentials( + AuthScope.ANY, + UsernamePasswordCredentials(username, password) + ) + httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider) + } + + httpClientBuilder + } + } + } + } +} diff --git a/backend/apps/dev/src/main/resources/config/application.yml b/backend/apps/dev/src/main/resources/config/application.yml index 8c578757ab..e46547c891 100644 --- a/backend/apps/dev/src/main/resources/config/application.yml +++ b/backend/apps/dev/src/main/resources/config/application.yml @@ -175,7 +175,14 @@ mailing: whitelistedDomains: sendRedirectedMailsTo: +opensearch: + uris: https://localhost:9200 + username: admin + password: 2cr8zkOoEXQJ3xUk!Aa1 + valtimo: + opensearch: + enabled: true app: scheme: http hostname: ${VALTIMO_APP_HOSTNAME} diff --git a/backend/apps/dev/src/main/resources/logback-spring.xml b/backend/apps/dev/src/main/resources/logback-spring.xml index e21b79db93..2197248e46 100644 --- a/backend/apps/dev/src/main/resources/logback-spring.xml +++ b/backend/apps/dev/src/main/resources/logback-spring.xml @@ -63,6 +63,7 @@ + true diff --git a/backend/apps/evenementenvergunning/src/main/resources/config/application.yml b/backend/apps/evenementenvergunning/src/main/resources/config/application.yml index 2d7fbfed62..9f8b884bee 100644 --- a/backend/apps/evenementenvergunning/src/main/resources/config/application.yml +++ b/backend/apps/evenementenvergunning/src/main/resources/config/application.yml @@ -137,6 +137,11 @@ server: mime-types: text/html,text/xml,text/plain,text/css, application/javascript, application/json min-response-size: 1024 +opensearch: + uris: ${OPENSEARCH_URIS:} + username: ${OPENSEARCH_USERNAME:} + password: ${OPENSEARCH_PASSWORD:} + mailing: onlyAllowWhitelistedRecipients: true redirectAllMails: false @@ -145,6 +150,8 @@ mailing: sendRedirectedMailsTo: valtimo: + opensearch: + enabled: ${VALTIMO_OPENSEARCH_ENABLED:false} app: scheme: http hostname: ${VALTIMO_APP_HOSTNAME} diff --git a/backend/apps/evenementenvergunning/src/main/resources/logback-spring.xml b/backend/apps/evenementenvergunning/src/main/resources/logback-spring.xml index 590e9819e8..4292a05559 100644 --- a/backend/apps/evenementenvergunning/src/main/resources/logback-spring.xml +++ b/backend/apps/evenementenvergunning/src/main/resources/logback-spring.xml @@ -1,5 +1,21 @@ + + @@ -44,6 +60,7 @@ + true diff --git a/backend/apps/gzac/src/main/resources/config/application.yml b/backend/apps/gzac/src/main/resources/config/application.yml index 2d7fbfed62..9f8b884bee 100644 --- a/backend/apps/gzac/src/main/resources/config/application.yml +++ b/backend/apps/gzac/src/main/resources/config/application.yml @@ -137,6 +137,11 @@ server: mime-types: text/html,text/xml,text/plain,text/css, application/javascript, application/json min-response-size: 1024 +opensearch: + uris: ${OPENSEARCH_URIS:} + username: ${OPENSEARCH_USERNAME:} + password: ${OPENSEARCH_PASSWORD:} + mailing: onlyAllowWhitelistedRecipients: true redirectAllMails: false @@ -145,6 +150,8 @@ mailing: sendRedirectedMailsTo: valtimo: + opensearch: + enabled: ${VALTIMO_OPENSEARCH_ENABLED:false} app: scheme: http hostname: ${VALTIMO_APP_HOSTNAME} diff --git a/backend/apps/gzac/src/main/resources/logback-spring.xml b/backend/apps/gzac/src/main/resources/logback-spring.xml index 590e9819e8..4292a05559 100644 --- a/backend/apps/gzac/src/main/resources/logback-spring.xml +++ b/backend/apps/gzac/src/main/resources/logback-spring.xml @@ -1,5 +1,21 @@ + + @@ -44,6 +60,7 @@ + true diff --git a/backend/apps/valtimo/src/main/resources/config/application.yml b/backend/apps/valtimo/src/main/resources/config/application.yml index e4ff90f8bd..4b9c346fcc 100644 --- a/backend/apps/valtimo/src/main/resources/config/application.yml +++ b/backend/apps/valtimo/src/main/resources/config/application.yml @@ -18,6 +18,8 @@ logging: management: endpoint: health: + probes: + enabled: true group: # readiness (and a dedicated startup group) only report UP once the bootstrap # health indicator is UP, i.e. after migrations + autodeployments have finished. @@ -137,6 +139,11 @@ server: mime-types: text/html,text/xml,text/plain,text/css, application/javascript, application/json min-response-size: 1024 +opensearch: + uris: ${OPENSEARCH_URIS:} + username: ${OPENSEARCH_USERNAME:} + password: ${OPENSEARCH_PASSWORD:} + mailing: onlyAllowWhitelistedRecipients: true redirectAllMails: false @@ -145,6 +152,8 @@ mailing: sendRedirectedMailsTo: valtimo: + opensearch: + enabled: ${VALTIMO_OPENSEARCH_ENABLED:false} app: scheme: http hostname: ${VALTIMO_APP_HOSTNAME} diff --git a/backend/apps/valtimo/src/main/resources/logback-spring.xml b/backend/apps/valtimo/src/main/resources/logback-spring.xml index 590e9819e8..4292a05559 100644 --- a/backend/apps/valtimo/src/main/resources/logback-spring.xml +++ b/backend/apps/valtimo/src/main/resources/logback-spring.xml @@ -1,5 +1,21 @@ + + @@ -44,6 +60,7 @@ + true diff --git a/backend/case-opensearch/build.gradle b/backend/case-opensearch/build.gradle new file mode 100644 index 0000000000..0cf3155a73 --- /dev/null +++ b/backend/case-opensearch/build.gradle @@ -0,0 +1,74 @@ +/* + * Copyright 2015-2024 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +dockerCompose { + projectName = "case-opensearch" + integrationTestingPostgresql { + isRequiredBy(project.tasks.integrationTestingPostgresql) + useComposeFiles.addAll( + "../docker-resources/docker-compose-base-test-postgresql.yml", + "docker-compose-override-postgresql.yml" + ) + } +} + +dependencies { + implementation project(":backend:admin-settings") + implementation project(":backend:authorization") + implementation project(":backend:case") + implementation project(":backend:inbox") + implementation project(":backend:outbox") + + // ShedLock for cross-instance coordination of the re-index job. The LockProvider bean itself + // is supplied at runtime by core's SchedulerAutoConfiguration; here we only need the API. + implementation "net.javacrumbs.shedlock:shedlock-spring:${shedlockVersion}" + + implementation "org.springframework.boot:spring-boot-starter-data-jpa" + implementation "org.springframework.boot:spring-boot-starter-web" + implementation "org.springframework.boot:spring-boot-starter-security" + implementation "org.springframework.boot:spring-boot-autoconfigure" + implementation "com.fasterxml.jackson.module:jackson-module-kotlin" + implementation "io.github.oshai:kotlin-logging:${kotlinLoggingVersion}" + + // OpenSearch via spring-data-opensearch (Apache 2.0 licensed) + implementation "org.opensearch.client:spring-data-opensearch-starter:${springDataOpenSearchVersion}" + + annotationProcessor "org.springframework.boot:spring-boot-autoconfigure-processor" + + testImplementation project(':backend:test-utils-common') + testImplementation project(':backend:core') + testImplementation(project(':backend:audit')) { + exclude(group: "com.ritense.valtimo", module: "case") + } + testImplementation "org.springframework.boot:spring-boot-starter-test" + testImplementation "org.mockito.kotlin:mockito-kotlin:${mockitoKotlinVersion}" + testImplementation "org.postgresql:postgresql" + testImplementation "org.springframework.security:spring-security-test" + + jar { + enabled = true + manifest { + attributes("Implementation-Title": "Ritense Case OpenSearch module") + attributes("Implementation-Version": projectVersion) + } + } +} + +tasks.named("integrationTestingPostgresql") { + systemProperty("liquibase.duplicateFileMode", "WARN") +} + +apply from: "gradle/publishing.gradle" diff --git a/backend/case-opensearch/docker-compose-override-postgresql.yml b/backend/case-opensearch/docker-compose-override-postgresql.yml new file mode 100644 index 0000000000..b1be670e23 --- /dev/null +++ b/backend/case-opensearch/docker-compose-override-postgresql.yml @@ -0,0 +1,20 @@ +services: + db: + ports: + - "3365:5432" + environment: + - POSTGRES_DB=case-opensearch-test + opensearch: + image: opensearchproject/opensearch:2.19.2 + environment: + - discovery.type=single-node + - DISABLE_SECURITY_PLUGIN=true + - DISABLE_INSTALL_DEMO_CONFIG=true + ports: + - "39200:9200" + healthcheck: + test: ["CMD-SHELL", "curl -sf http://localhost:9200/_cluster/health || exit 1"] + interval: 5s + timeout: 10s + retries: 40 + start_period: 15s diff --git a/backend/case-opensearch/gradle/publishing.gradle b/backend/case-opensearch/gradle/publishing.gradle new file mode 100644 index 0000000000..f991a5d60f --- /dev/null +++ b/backend/case-opensearch/gradle/publishing.gradle @@ -0,0 +1,35 @@ +/* + * Copyright 2015-2024 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +pluginManager.withPlugin('maven-publish') { + publishing { + publications { + maven(MavenPublication) { + pom { + name = 'Case OpenSearch module' + description = 'The case-opensearch module syncs json_schema_document to OpenSearch as a CQRS read model' + developers { + developer { + id = "team-valtimo" + name = "Team Valtimo" + email = "team-valtimo@ritense.com" + } + } + } + } + } + } +} diff --git a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/OpenSearchProperties.kt b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/OpenSearchProperties.kt new file mode 100644 index 0000000000..8bfb7af614 --- /dev/null +++ b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/OpenSearchProperties.kt @@ -0,0 +1,72 @@ +/* + * Copyright 2015-2026 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.ritense.document.opensearch + +import org.springframework.boot.context.properties.ConfigurationProperties +import org.springframework.boot.context.properties.NestedConfigurationProperty +import java.time.Duration + +@ConfigurationProperties(prefix = "valtimo.opensearch") +data class OpenSearchProperties( + val enabled: Boolean = false, + val healthCheckEnabled: Boolean = true, + val healthCheckIntervalMs: Long = 30000, + val fallbackWarningIntervalMs: Long = 300000, + + @NestedConfigurationProperty + val reconcile: Reconcile = Reconcile(), + + @NestedConfigurationProperty + val reindex: Reindex = Reindex(), +) { + /** + * Configuration for the self-healing reconciler that keeps the OpenSearch index in sync with + * PostgreSQL as a derived read-model. + * + * @property enabled whether the scheduled reconcile job runs at all. + * @property interval delay between the end of one reconcile cycle and the start of the next + * (also bound directly by the job's `@Scheduled(fixedDelayString)`). + * @property overlap δ subtracted from the watermark each cycle to safely cover the + * flush→commit boundary; re-indexes a small trailing window (idempotent). + * @property pageSize DB keyset page size for the incremental upsert scan. + * @property pendingDeletionBatchSize number of pending index deletions drained per batch. + */ + data class Reconcile( + val enabled: Boolean = true, + // Keep in sync with the @Scheduled(fixedDelayString) default in DocumentOpenSearchReconcileJob (PT2M). + val interval: Duration = Duration.ofMinutes(2), + val overlap: Duration = Duration.ofSeconds(10), + val pageSize: Int = 5000, + val pendingDeletionBatchSize: Int = 500, + ) + + /** + * Behaviour while an admin (re)index run is filling the index. + * + * @property fallbackToPostgresWhileRunning while a reindex run is in progress, route document search + * to PostgreSQL so users never query a partially-filled index; search returns to OpenSearch + * automatically once all runs finish. Does not affect the reconciler (which keeps the index + * complete) — only the admin reindex. + * @property runningHeartbeatTimeout a RUNNING run is only treated as in-progress while its heartbeat + * is fresher than this. Guards against a run left behind by a crashed instance pinning search + * to PostgreSQL indefinitely. Must exceed the longest expected gap between reindex batches. + */ + data class Reindex( + val fallbackToPostgresWhileRunning: Boolean = true, + val runningHeartbeatTimeout: Duration = Duration.ofMinutes(5), + ) +} diff --git a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/authorization/OpenSearchAuthorizationEntityMapper.kt b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/authorization/OpenSearchAuthorizationEntityMapper.kt new file mode 100644 index 0000000000..e6cffef10d --- /dev/null +++ b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/authorization/OpenSearchAuthorizationEntityMapper.kt @@ -0,0 +1,41 @@ +/* + * Copyright 2015-2024 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.ritense.document.opensearch.authorization + +import com.ritense.authorization.permission.condition.PermissionCondition +import org.opensearch.index.query.QueryBuilder + +/** + * OpenSearch equivalent of [com.ritense.authorization.AuthorizationEntityMapper]. + * + * Translates a [com.ritense.authorization.permission.condition.ContainerPermissionCondition] + * on entity type [TO] into an OpenSearch [QueryBuilder] that filters [FROM] documents. + * + * Implement this interface and register the implementation as a Spring bean to add support + * for a new container relationship without modifying the core translator. + */ +interface OpenSearchAuthorizationEntityMapper { + + /** + * Given conditions on the [TO] entity type, returns an OpenSearch [QueryBuilder] that filters + * [FROM] documents satisfying those conditions, or `null` if no filter is needed + * (i.e. any [FROM] document qualifies regardless of [conditions]). + */ + fun mapQuery(conditions: List): QueryBuilder? + + fun supports(fromClass: Class<*>, toClass: Class<*>): Boolean +} diff --git a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/authorization/OpenSearchPermissionConditionTranslator.kt b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/authorization/OpenSearchPermissionConditionTranslator.kt new file mode 100644 index 0000000000..1c9ed6bfcc --- /dev/null +++ b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/authorization/OpenSearchPermissionConditionTranslator.kt @@ -0,0 +1,225 @@ +/* + * Copyright 2015-2024 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.ritense.document.opensearch.authorization + +import com.ritense.authorization.Action +import com.ritense.authorization.AuthorizationContext.Companion.runWithoutAuthorization +import com.ritense.authorization.AuthorizationService +import com.ritense.authorization.permission.ConditionContainer +import com.ritense.authorization.permission.Permission +import com.ritense.authorization.permission.condition.ContainerPermissionCondition +import com.ritense.authorization.permission.condition.ExpressionPermissionCondition +import com.ritense.authorization.permission.condition.FieldPermissionCondition +import com.ritense.authorization.permission.condition.PermissionCondition +import com.ritense.authorization.permission.condition.PermissionConditionOperator +import com.ritense.authorization.request.EntityAuthorizationRequest +import com.ritense.authorization.role.Role +import com.ritense.document.domain.impl.JsonSchemaDocument +import com.ritense.document.repository.impl.JsonSchemaDocumentRepository +import com.ritense.valtimo.contract.authorization.CurrentUserExpressionHandler +import io.github.oshai.kotlinlogging.KotlinLogging +import org.opensearch.index.query.BoolQueryBuilder +import org.opensearch.index.query.QueryBuilder +import org.opensearch.index.query.QueryBuilders + +class OpenSearchPermissionConditionTranslator( + private val openSearchMappers: List>, + private val authorizationService: AuthorizationService, + private val documentRepository: JsonSchemaDocumentRepository, +) { + + /** + * Translates a list of [Permission]s into a single OpenSearch [QueryBuilder] that, when applied + * to a search, returns only the documents the current user is allowed to see for [action]. + * + * Permissions are OR-ed; conditions within a permission are AND-ed. + * Returns a deny-all query if no permissions match [action]. + */ + fun toQuery(permissions: List, action: Action<*>): QueryBuilder { + val matching = permissions.filter { + it.resourceType == JsonSchemaDocument::class.java && it.actions.contains(action) + } + logger.debug { "toQuery: ${permissions.size} permissions total, ${matching.size} matching action=$action" } + if (matching.isEmpty()) { + return denyAll() + } + + val perPermissionQueries = matching.map { permission -> + val conditionQueries = permission.conditionContainer.conditions.map { translateCondition(it) } + andAll(conditionQueries) + } + val result = if (perPermissionQueries.size == 1) { + perPermissionQueries.first() + } else { + QueryBuilders.boolQuery().apply { + perPermissionQueries.forEach { should(it) } + minimumShouldMatch(1) + } + } + logger.debug { "toQuery: generated query for action=$action" } + return result + } + + private fun translateCondition(condition: PermissionCondition): QueryBuilder = when (condition) { + is FieldPermissionCondition<*> -> translateField(condition) + is ExpressionPermissionCondition<*> -> translateExpression(condition) + is ContainerPermissionCondition<*> -> translateContainer(condition) + else -> throw IllegalArgumentException("Unknown permission condition type: ${condition::class.qualifiedName}") + } + + private fun translateField(cond: FieldPermissionCondition<*>): QueryBuilder { + val baseField = jpaToOsField(cond.field) + val value = resolveFieldValue(cond) + val osField = if (isDynamicTextField(baseField, cond.operator, value)) "$baseField.keyword" else baseField + return Companion.applyOperator(osField, cond.operator, value) + } + + private fun translateExpression(cond: ExpressionPermissionCondition<*>): QueryBuilder { + val dotPath = cond.path.removePrefix("$.").replace("/", ".") + val baseField = "${jpaToOsField(cond.field)}.$dotPath" + val value = CurrentUserExpressionHandler.resolveValue(cond.value) + // Content sub-fields are dynamically mapped as text — use .keyword for string term queries + val osField = if (isDynamicTextField(baseField, cond.operator, value)) "$baseField.keyword" else baseField + logger.debug { "translateExpression: field=${cond.field} → osField=$osField, op=${cond.operator}, value=$value (${value?.javaClass?.simpleName})" } + return Companion.applyOperator(osField, cond.operator, value) + } + + @Suppress("UNCHECKED_CAST") + private fun translateContainer(cond: ContainerPermissionCondition<*>): QueryBuilder { + val osMapper = openSearchMappers.find { + it.supports(JsonSchemaDocument::class.java, cond.resourceType) + } as? OpenSearchAuthorizationEntityMapper + + if (osMapper != null) { + return osMapper.mapQuery(cond.conditions) ?: noFilter() + } + + logger.warn { + "No OpenSearchAuthorizationEntityMapper registered for " + + "JsonSchemaDocument → ${cond.resourceType.simpleName}. " + + "Falling back to JPA ID resolution — may be slow for large datasets." + } + return jpaFallback(cond) + } + + /** + * Fallback for [ContainerPermissionCondition] types that have no registered + * [OpenSearchAuthorizationEntityMapper]. Uses JPA to find matching document IDs and + * returns an `ids` query. + */ + private fun jpaFallback(cond: ContainerPermissionCondition<*>): QueryBuilder { + val syntheticPermission = Permission( + resourceType = JsonSchemaDocument::class.java, + actions = mutableListOf(Action(Action.IGNORE)), + conditionContainer = ConditionContainer(listOf(cond)), + role = Role(key = ""), + ) + val spec = authorizationService.getAuthorizationSpecification( + EntityAuthorizationRequest(JsonSchemaDocument::class.java, Action(Action.IGNORE)), + listOf(syntheticPermission) + ) + val allowedIds: List = runWithoutAuthorization { + documentRepository.findAll(spec).map { doc -> doc.id().toString() } + } + return QueryBuilders.idsQuery().addIds(*allowedIds.toTypedArray()) + } + + private fun resolveFieldValue(cond: FieldPermissionCondition<*>): Any? = + if (cond.value is List<*>) { + (cond.value as List<*>).map { CurrentUserExpressionHandler.resolveValue(it) } + } else { + CurrentUserExpressionHandler.resolveValue(cond.value) + } + + companion object { + private val logger = KotlinLogging.logger {} + + fun applyOperator(field: String, op: PermissionConditionOperator, value: Any?): QueryBuilder = + when (op) { + PermissionConditionOperator.EQUAL_TO -> { + if (value == null) { + QueryBuilders.boolQuery().mustNot(QueryBuilders.existsQuery(field)) + } else { + QueryBuilders.termQuery(field, value) + } + } + PermissionConditionOperator.NOT_EQUAL_TO -> { + if (value == null) { + QueryBuilders.existsQuery(field) + } else { + QueryBuilders.boolQuery().mustNot(QueryBuilders.termQuery(field, value)) + } + } + PermissionConditionOperator.GREATER_THAN -> + QueryBuilders.rangeQuery(field).gt(value) + PermissionConditionOperator.GREATER_THAN_OR_EQUAL_TO -> + QueryBuilders.rangeQuery(field).gte(value) + PermissionConditionOperator.LESS_THAN -> + QueryBuilders.rangeQuery(field).lt(value) + PermissionConditionOperator.LESS_THAN_OR_EQUAL_TO -> + QueryBuilders.rangeQuery(field).lte(value) + PermissionConditionOperator.LIST_CONTAINS -> + QueryBuilders.termQuery(field, value) + PermissionConditionOperator.IN -> { + val collection = value as? Collection<*> + ?: throw IllegalArgumentException("IN operator requires a Collection value") + QueryBuilders.termsQuery(field, collection.toList()) + } + } + + /** + * Maps JPA entity field names (as used in [FieldPermissionCondition.field]) to + * their corresponding field names in the OpenSearch document. + */ + val fieldMappings: Map = mapOf( + "createdBy" to "createdBy", + "assigneeId" to "assigneeId", + "assigneeFullName" to "assigneeFullName", + "content" to "content", + "content.content" to "content", + "sequence" to "sequence", + "retentionDate" to "retentionDate", + ) + + fun jpaToOsField(jpaField: String): String = fieldMappings[jpaField] ?: jpaField + + /** + * Content sub-fields use dynamic mapping (text + keyword). String term queries + * (EQUAL_TO, NOT_EQUAL_TO, LIST_CONTAINS, IN) need the .keyword sub-field for exact match. + */ + fun isDynamicTextField(field: String, op: PermissionConditionOperator, value: Any?): Boolean { + if (!field.startsWith("content.")) return false + if (value == null) return false + val isStringValue = value is String || (value is Collection<*> && value.firstOrNull() is String) + val isTermOp = op in setOf( + PermissionConditionOperator.EQUAL_TO, + PermissionConditionOperator.NOT_EQUAL_TO, + PermissionConditionOperator.LIST_CONTAINS, + PermissionConditionOperator.IN, + ) + return isStringValue && isTermOp + } + + fun denyAll(): QueryBuilder = QueryBuilders.idsQuery() + fun noFilter(): QueryBuilder = QueryBuilders.matchAllQuery() + fun andAll(list: List): QueryBuilder = when { + list.isEmpty() -> noFilter() + list.size == 1 -> list.first() + else -> QueryBuilders.boolQuery().apply { list.forEach { must(it) } } + } + } +} diff --git a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/authorization/mapper/JsonSchemaDocumentCaseDefinitionOpenSearchMapper.kt b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/authorization/mapper/JsonSchemaDocumentCaseDefinitionOpenSearchMapper.kt new file mode 100644 index 0000000000..1673076c67 --- /dev/null +++ b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/authorization/mapper/JsonSchemaDocumentCaseDefinitionOpenSearchMapper.kt @@ -0,0 +1,71 @@ +/* + * Copyright 2015-2026 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.ritense.document.opensearch.authorization.mapper + +import com.ritense.authorization.permission.condition.FieldPermissionCondition +import com.ritense.authorization.permission.condition.PermissionCondition +import com.ritense.case_.domain.definition.CaseDefinition +import com.ritense.document.domain.impl.JsonSchemaDocument +import com.ritense.document.opensearch.authorization.OpenSearchAuthorizationEntityMapper +import com.ritense.document.opensearch.authorization.OpenSearchPermissionConditionTranslator.Companion.andAll +import com.ritense.document.opensearch.authorization.OpenSearchPermissionConditionTranslator.Companion.applyOperator +import com.ritense.valtimo.contract.authorization.CurrentUserExpressionHandler +import org.opensearch.index.query.QueryBuilder +import org.opensearch.index.query.QueryBuilders + +/** + * Handles [com.ritense.authorization.permission.condition.ContainerPermissionCondition] + * where the container resource type is [CaseDefinition]. + * + * Field paths mirror the JPA entity: `definitionId.blueprintId.*`. + */ +class JsonSchemaDocumentCaseDefinitionOpenSearchMapper : OpenSearchAuthorizationEntityMapper { + + override fun mapQuery(conditions: List): QueryBuilder? { + val conditionQueries = conditions.map { condition -> + when (condition) { + is FieldPermissionCondition<*> -> { + val osField = mapCaseDefinitionField(condition.field) + val value = CurrentUserExpressionHandler.resolveValue(condition.value) + applyOperator(osField, condition.operator, value) + } + else -> throw UnsupportedOperationException( + "Condition type ${condition::class.simpleName} is not supported in " + + "${this::class.simpleName}. Register a custom ${OpenSearchAuthorizationEntityMapper::class.simpleName} " + + "or extend this mapper to handle it." + ) + } + } + + // Constrain to CASE blueprint type to exclude BUILDING_BLOCK documents + val typeQuery = QueryBuilders.termQuery("definitionId.blueprintId.blueprintType", "CASE") + return andAll(conditionQueries + typeQuery) + } + + override fun supports(fromClass: Class<*>, toClass: Class<*>): Boolean = + fromClass == JsonSchemaDocument::class.java && + toClass == CaseDefinition::class.java + + private fun mapCaseDefinitionField(field: String): String = when (field) { + "id.key" -> "definitionId.blueprintId.blueprintKey" + "id.versionTag" -> "definitionId.blueprintId.blueprintVersionTag" + else -> throw UnsupportedOperationException( + "Field '$field' on CaseDefinition is not yet mapped for OpenSearch. " + + "Add it to ${this::class.simpleName}." + ) + } +} diff --git a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/authorization/mapper/JsonSchemaDocumentDefinitionOpenSearchMapper.kt b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/authorization/mapper/JsonSchemaDocumentDefinitionOpenSearchMapper.kt new file mode 100644 index 0000000000..9a274dbe57 --- /dev/null +++ b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/authorization/mapper/JsonSchemaDocumentDefinitionOpenSearchMapper.kt @@ -0,0 +1,69 @@ +/* + * Copyright 2015-2026 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.ritense.document.opensearch.authorization.mapper + +import com.ritense.authorization.permission.condition.FieldPermissionCondition +import com.ritense.authorization.permission.condition.PermissionCondition +import com.ritense.document.domain.impl.JsonSchemaDocument +import com.ritense.document.domain.impl.JsonSchemaDocumentDefinition +import com.ritense.document.opensearch.authorization.OpenSearchAuthorizationEntityMapper +import com.ritense.document.opensearch.authorization.OpenSearchPermissionConditionTranslator.Companion.andAll +import com.ritense.document.opensearch.authorization.OpenSearchPermissionConditionTranslator.Companion.applyOperator +import com.ritense.valtimo.contract.authorization.CurrentUserExpressionHandler +import org.opensearch.index.query.QueryBuilder + +/** + * Handles [com.ritense.authorization.permission.condition.ContainerPermissionCondition] + * where the container resource type is [JsonSchemaDocumentDefinition]. + * + * Field paths mirror the JPA entity: `definitionId.name` and `definitionId.version`. + */ +class JsonSchemaDocumentDefinitionOpenSearchMapper : OpenSearchAuthorizationEntityMapper { + + override fun mapQuery(conditions: List): QueryBuilder? { + if (conditions.isEmpty()) return null + + val queries = conditions.map { condition -> + when (condition) { + is FieldPermissionCondition<*> -> { + val osField = mapDefinitionField(condition.field) + val value = CurrentUserExpressionHandler.resolveValue(condition.value) + applyOperator(osField, condition.operator, value) + } + else -> throw UnsupportedOperationException( + "Condition type ${condition::class.simpleName} is not supported in " + + "${this::class.simpleName}. Register a custom ${OpenSearchAuthorizationEntityMapper::class.simpleName} " + + "or extend this mapper to handle it." + ) + } + } + return andAll(queries) + } + + override fun supports(fromClass: Class<*>, toClass: Class<*>): Boolean = + fromClass == JsonSchemaDocument::class.java && + toClass == JsonSchemaDocumentDefinition::class.java + + private fun mapDefinitionField(field: String): String = when (field) { + "id.name" -> "definitionId.name" + "id.version" -> "definitionId.version" + else -> throw UnsupportedOperationException( + "Field '$field' on JsonSchemaDocumentDefinition is not yet mapped for OpenSearch. " + + "Add it to ${this::class.simpleName}." + ) + } +} diff --git a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/autoconfigure/DocumentOpenSearchAutoConfiguration.kt b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/autoconfigure/DocumentOpenSearchAutoConfiguration.kt new file mode 100644 index 0000000000..7c99d2d5fe --- /dev/null +++ b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/autoconfigure/DocumentOpenSearchAutoConfiguration.kt @@ -0,0 +1,381 @@ +/* + * Copyright 2015-2026 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.ritense.document.opensearch.autoconfigure + +import com.fasterxml.jackson.databind.ObjectMapper +import io.github.oshai.kotlinlogging.KotlinLogging +import com.ritense.adminsettings.service.FeatureToggleOverridesService +import com.ritense.authorization.AuthorizationService +import com.ritense.document.opensearch.OpenSearchProperties +import com.ritense.document.autoconfigure.DocumentAutoConfiguration +import com.ritense.document.opensearch.authorization.OpenSearchAuthorizationEntityMapper +import com.ritense.document.opensearch.authorization.OpenSearchPermissionConditionTranslator +import com.ritense.document.opensearch.authorization.mapper.JsonSchemaDocumentCaseDefinitionOpenSearchMapper +import com.ritense.document.opensearch.authorization.mapper.JsonSchemaDocumentDefinitionOpenSearchMapper +import com.ritense.document.opensearch.domain.OpenSearchReindexRun +import com.ritense.document.opensearch.handler.DocumentOpenSearchEventListener +import com.ritense.document.opensearch.handler.PendingIndexDeletionListener +import com.ritense.document.opensearch.repository.JsonSchemaDocumentOpenSearchRepository +import com.ritense.document.opensearch.repository.OpenSearchReconcileStateRepository +import com.ritense.document.opensearch.repository.OpenSearchReindexRunRepository +import com.ritense.document.opensearch.repository.PendingIndexDeletionRepository +import com.ritense.document.opensearch.security.DocumentOpenSearchHttpSecurityConfigurer +import com.ritense.document.opensearch.service.DelegatingDocumentSearchService +import com.ritense.document.opensearch.service.DocumentOpenSearchQueryService +import com.ritense.document.opensearch.service.DocumentOpenSearchReconcileJob +import com.ritense.document.opensearch.service.DocumentOpenSearchIndexInitializer +import com.ritense.document.opensearch.service.DocumentOpenSearchReconcileService +import com.ritense.document.opensearch.service.DocumentOpenSearchReindexService +import com.ritense.document.opensearch.service.DocumentOpenSearchSyncService +import com.ritense.document.opensearch.service.JsonSchemaDocumentOpenSearchService +import com.ritense.document.opensearch.service.JsonSchemaDocumentOsConverter +import com.ritense.document.opensearch.service.OpenSearchReindexRunService +import com.ritense.document.opensearch.service.ReindexProgressGate +import com.ritense.document.opensearch.service.OpenSearchHealthService +import com.ritense.document.opensearch.service.SearchEngineToggle +import com.ritense.document.opensearch.web.DocumentOpenSearchReindexResource +import com.ritense.document.opensearch.web.SearchEngineResource +import com.ritense.document.repository.impl.JsonSchemaDocumentRepository +import com.ritense.document.service.DocumentSearchService +import com.ritense.document.service.impl.JsonSchemaDocumentDefinitionService +import com.ritense.document.service.SearchFieldService +import com.ritense.document.service.impl.JsonSchemaDocumentSearchService +import com.ritense.case.service.CaseDefinitionService +import com.ritense.valtimo.contract.database.QueryDialectHelper +import com.ritense.outbox.OutboxService +import com.ritense.valtimo.contract.authentication.TeamManagementService +import com.ritense.valtimo.contract.authentication.UserManagementService +import jakarta.persistence.EntityManager +import net.javacrumbs.shedlock.core.LockProvider +import org.springframework.boot.ApplicationRunner +import org.springframework.boot.autoconfigure.AutoConfiguration +import org.springframework.boot.autoconfigure.AutoConfigureBefore +import org.springframework.boot.autoconfigure.condition.ConditionalOnBean +import org.springframework.boot.autoconfigure.condition.ConditionalOnClass +import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty +import org.springframework.boot.context.properties.EnableConfigurationProperties +import org.springframework.boot.autoconfigure.domain.EntityScan +import org.springframework.context.annotation.Bean +import org.springframework.core.annotation.Order +import org.springframework.data.elasticsearch.core.ElasticsearchOperations +import org.springframework.data.elasticsearch.repository.config.EnableElasticsearchRepositories +import org.springframework.data.jpa.repository.config.EnableJpaRepositories +import org.springframework.transaction.PlatformTransactionManager +import org.springframework.scheduling.annotation.EnableScheduling +import org.springframework.scheduling.annotation.Scheduled + +@AutoConfiguration +@AutoConfigureBefore(DocumentAutoConfiguration::class) +@ConditionalOnClass(ElasticsearchOperations::class) +@EnableScheduling +@EnableElasticsearchRepositories(basePackages = ["com.ritense.document.opensearch.repository"]) +@EnableConfigurationProperties(OpenSearchProperties::class) +@EnableJpaRepositories(basePackageClasses = [OpenSearchReindexRunRepository::class]) +@EntityScan(basePackageClasses = [OpenSearchReindexRun::class]) +class DocumentOpenSearchAutoConfiguration { + + @Bean + @ConditionalOnMissingBean + fun jsonSchemaDocumentDefinitionOpenSearchMapper(): JsonSchemaDocumentDefinitionOpenSearchMapper = + JsonSchemaDocumentDefinitionOpenSearchMapper() + + @Bean + @ConditionalOnMissingBean + fun jsonSchemaDocumentCaseDefinitionOpenSearchMapper(): JsonSchemaDocumentCaseDefinitionOpenSearchMapper = + JsonSchemaDocumentCaseDefinitionOpenSearchMapper() + + @Bean + @ConditionalOnMissingBean + fun openSearchPermissionConditionTranslator( + openSearchMappers: List>, + authorizationService: AuthorizationService, + documentRepository: JsonSchemaDocumentRepository, + ): OpenSearchPermissionConditionTranslator = + OpenSearchPermissionConditionTranslator(openSearchMappers, authorizationService, documentRepository) + + @Bean + @ConditionalOnMissingBean + fun documentOpenSearchQueryService( + elasticsearchOperations: ElasticsearchOperations, + authorizationService: AuthorizationService, + translator: OpenSearchPermissionConditionTranslator, + ): DocumentOpenSearchQueryService = + DocumentOpenSearchQueryService(elasticsearchOperations, authorizationService, translator) + + @Bean + @ConditionalOnMissingBean + fun jsonSchemaDocumentOsConverter( + objectMapper: ObjectMapper, + openSearchRepository: JsonSchemaDocumentOpenSearchRepository, + ): JsonSchemaDocumentOsConverter = + JsonSchemaDocumentOsConverter(objectMapper, openSearchRepository) + + @Bean + @ConditionalOnMissingBean + fun documentOpenSearchSyncService( + repository: JsonSchemaDocumentOpenSearchRepository, + documentRepository: JsonSchemaDocumentRepository, + converter: JsonSchemaDocumentOsConverter, + transactionManager: PlatformTransactionManager, + ): DocumentOpenSearchSyncService = + DocumentOpenSearchSyncService(repository, documentRepository, converter, transactionManager) + + @Bean + @ConditionalOnProperty(prefix = "valtimo.opensearch", name = ["enabled"], havingValue = "true", matchIfMissing = false) + fun documentOpenSearchEventListener( + syncService: DocumentOpenSearchSyncService, + searchEngineToggle: SearchEngineToggle, + ): DocumentOpenSearchEventListener = + DocumentOpenSearchEventListener(syncService, searchEngineToggle) + + @Bean + @ConditionalOnMissingBean + fun openSearchReindexRunService( + openSearchReindexRunRepository: OpenSearchReindexRunRepository, + objectMapper: ObjectMapper, + openSearchProperties: OpenSearchProperties, + entityManager: EntityManager, + ): OpenSearchReindexRunService = + OpenSearchReindexRunService(openSearchReindexRunRepository, objectMapper, openSearchProperties, entityManager) + + @Bean + @ConditionalOnMissingBean + fun documentOpenSearchReindexService( + entityManager: EntityManager, + converter: JsonSchemaDocumentOsConverter, + elasticsearchOperations: ElasticsearchOperations, + transactionManager: PlatformTransactionManager, + lockProvider: LockProvider, + openSearchReindexRunService: OpenSearchReindexRunService, + openSearchRepository: JsonSchemaDocumentOpenSearchRepository, + ): DocumentOpenSearchReindexService = + DocumentOpenSearchReindexService( + entityManager, + converter, + elasticsearchOperations, + transactionManager, + lockProvider, + openSearchReindexRunService, + openSearchRepository, + ) + + @Bean + @ConditionalOnMissingBean + @ConditionalOnProperty(prefix = "valtimo.opensearch", name = ["enabled"], havingValue = "true", matchIfMissing = false) + fun documentOpenSearchReconcileService( + entityManager: EntityManager, + converter: JsonSchemaDocumentOsConverter, + openSearchRepository: JsonSchemaDocumentOpenSearchRepository, + reconcileStateRepository: OpenSearchReconcileStateRepository, + pendingIndexDeletionRepository: PendingIndexDeletionRepository, + transactionManager: PlatformTransactionManager, + lockProvider: LockProvider, + openSearchProperties: OpenSearchProperties, + ): DocumentOpenSearchReconcileService = + DocumentOpenSearchReconcileService( + entityManager, + converter, + openSearchRepository, + reconcileStateRepository, + pendingIndexDeletionRepository, + transactionManager, + lockProvider, + openSearchProperties, + ) + + @Bean + @ConditionalOnMissingBean + @ConditionalOnBean(DocumentOpenSearchReconcileService::class) + @ConditionalOnProperty(prefix = "valtimo.opensearch.reconcile", name = ["enabled"], havingValue = "true", matchIfMissing = true) + fun documentOpenSearchReconcileJob( + reconcileService: DocumentOpenSearchReconcileService, + searchEngineToggle: SearchEngineToggle, + ): DocumentOpenSearchReconcileJob = + DocumentOpenSearchReconcileJob(reconcileService, searchEngineToggle) + + @Bean + @ConditionalOnMissingBean + @ConditionalOnProperty(prefix = "valtimo.opensearch", name = ["enabled"], havingValue = "true", matchIfMissing = false) + fun pendingIndexDeletionListener( + pendingIndexDeletionRepository: PendingIndexDeletionRepository, + ): PendingIndexDeletionListener = + PendingIndexDeletionListener(pendingIndexDeletionRepository) + + @Order(294) + @Bean + @ConditionalOnMissingBean + fun documentOpenSearchHttpSecurityConfigurer(): DocumentOpenSearchHttpSecurityConfigurer = + DocumentOpenSearchHttpSecurityConfigurer() + + // --- Search engine toggle: both implementations + delegating service --- + + @Bean + @ConditionalOnMissingBean + // Start in POSTGRES so no OpenSearch call happens before searchEngineSettingLoader resolves the real + // engine. The @Scheduled reconciler is armed during context refresh, before ApplicationRunners run, so + // an OPENSEARCH default could otherwise leak one spurious call at startup even when the engine is off. + fun searchEngineToggle(): SearchEngineToggle = SearchEngineToggle(SearchEngineToggle.Engine.POSTGRES) + + @Bean + @ConditionalOnMissingBean + fun reindexProgressGate( + openSearchReindexRunService: OpenSearchReindexRunService, + openSearchProperties: OpenSearchProperties, + ): ReindexProgressGate = + ReindexProgressGate(openSearchReindexRunService, openSearchProperties) + + @Bean("openSearchDocumentSearchService") + fun openSearchDocumentSearchService( + elasticsearchOperations: ElasticsearchOperations, + translator: OpenSearchPermissionConditionTranslator, + authorizationService: AuthorizationService, + jpaRepository: JsonSchemaDocumentRepository, + userManagementService: UserManagementService, + searchFieldService: SearchFieldService, + outboxService: OutboxService, + objectMapper: ObjectMapper, + caseDefinitionService: CaseDefinitionService, + ): JsonSchemaDocumentOpenSearchService = + JsonSchemaDocumentOpenSearchService( + elasticsearchOperations, translator, authorizationService, + jpaRepository, userManagementService, searchFieldService, outboxService, objectMapper, + caseDefinitionService, + ) + + @Bean("jpaDocumentSearchService") + fun jpaDocumentSearchService( + entityManager: EntityManager, + queryDialectHelper: QueryDialectHelper, + searchFieldService: SearchFieldService, + userManagementService: UserManagementService, + teamManagementService: TeamManagementService, + authorizationService: AuthorizationService, + outboxService: OutboxService, + jsonSchemaDocumentDefinitionService: JsonSchemaDocumentDefinitionService, + objectMapper: ObjectMapper, + ): JsonSchemaDocumentSearchService = + JsonSchemaDocumentSearchService( + entityManager, queryDialectHelper, searchFieldService, + userManagementService, teamManagementService, authorizationService, outboxService, + jsonSchemaDocumentDefinitionService, objectMapper, + ) + + @Bean + @org.springframework.context.annotation.Primary + fun documentSearchService( + openSearchDocumentSearchService: JsonSchemaDocumentOpenSearchService, + jpaDocumentSearchService: JsonSchemaDocumentSearchService, + searchEngineToggle: SearchEngineToggle, + reindexProgressGate: ReindexProgressGate, + ): DelegatingDocumentSearchService = + DelegatingDocumentSearchService( + openSearchDocumentSearchService, jpaDocumentSearchService, searchEngineToggle, reindexProgressGate, + ) + + @Bean + @ConditionalOnMissingBean + fun searchEngineResource( + toggle: SearchEngineToggle, + openSearchProperties: OpenSearchProperties, + featureToggleOverridesService: FeatureToggleOverridesService, + indexInitializer: DocumentOpenSearchIndexInitializer, + ): SearchEngineResource = + SearchEngineResource(toggle, openSearchProperties, featureToggleOverridesService, indexInitializer) + + @Bean + @ConditionalOnMissingBean + fun documentOpenSearchReindexResource( + reindexService: DocumentOpenSearchReindexService, + ): DocumentOpenSearchReindexResource = + DocumentOpenSearchReindexResource(reindexService) + + @Bean + @ConditionalOnMissingBean + fun documentOpenSearchIndexInitializer( + elasticsearchOperations: ElasticsearchOperations, + ): DocumentOpenSearchIndexInitializer = + DocumentOpenSearchIndexInitializer(elasticsearchOperations) + + /** + * Resolves the active search engine on startup from configuration and the persisted feature-toggle + * override, then — only when OpenSearch is the active engine — provisions the index. Merged into a + * single ordered runner so the toggle is always set before any index/OpenSearch work is decided, and + * so a disabled or toggled-off engine performs no active OpenSearch call at all on boot. + */ + @Bean + fun searchEngineSettingLoader( + toggle: SearchEngineToggle, + featureToggleOverridesService: FeatureToggleOverridesService, + openSearchProperties: OpenSearchProperties, + indexInitializer: DocumentOpenSearchIndexInitializer, + ): ApplicationRunner = ApplicationRunner { + if (!openSearchProperties.enabled) { + toggle.set(SearchEngineToggle.Engine.POSTGRES) + logger.info { "OpenSearch disabled via configuration; using PostgreSQL for document search" } + return@ApplicationRunner + } + + val overrides = featureToggleOverridesService.getOverrides().overrides + val useOpenSearch = overrides[SEARCH_ENGINE_TOGGLE_KEY] ?: true + val engine = if (useOpenSearch) SearchEngineToggle.Engine.OPENSEARCH else SearchEngineToggle.Engine.POSTGRES + toggle.set(engine) + logger.info { "Document search engine set to: ${engine.name}" } + + if (toggle.isOpenSearchActive()) { + try { + indexInitializer.ensureIndex() + } catch (e: Exception) { + logger.warn(e) { "Failed to initialize OpenSearch index at startup — is OpenSearch running?" } + } + } + } + + @Bean + @ConditionalOnMissingBean + @ConditionalOnProperty(prefix = "valtimo.opensearch", name = ["enabled"], havingValue = "true", matchIfMissing = false) + @ConditionalOnProperty(prefix = "valtimo.opensearch", name = ["health-check-enabled"], havingValue = "true", matchIfMissing = true) + fun openSearchHealthService( + restHighLevelClient: org.opensearch.client.RestHighLevelClient, + toggle: SearchEngineToggle, + openSearchProperties: OpenSearchProperties, + ): OpenSearchHealthService = + OpenSearchHealthService(restHighLevelClient, toggle, openSearchProperties) + + @Bean + @ConditionalOnBean(OpenSearchHealthService::class) + fun openSearchHealthScheduler( + healthService: OpenSearchHealthService, + openSearchProperties: OpenSearchProperties, + ): OpenSearchHealthScheduler = + OpenSearchHealthScheduler(healthService, openSearchProperties) + + companion object { + private val logger = KotlinLogging.logger {} + const val SEARCH_ENGINE_TOGGLE_KEY = "useOpenSearchForDocumentSearch" + } +} + +class OpenSearchHealthScheduler( + private val healthService: OpenSearchHealthService, + private val properties: OpenSearchProperties, +) { + @Scheduled(fixedDelayString = "\${valtimo.opensearch.health-check-interval-ms:30000}") + fun checkHealth() { + healthService.checkAndRecover() + } +} diff --git a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/autoconfigure/ExcludeElasticsearchAutoConfigurationFilter.kt b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/autoconfigure/ExcludeElasticsearchAutoConfigurationFilter.kt new file mode 100644 index 0000000000..3bfa920f4a --- /dev/null +++ b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/autoconfigure/ExcludeElasticsearchAutoConfigurationFilter.kt @@ -0,0 +1,56 @@ +/* + * Copyright 2015-2026 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.ritense.document.opensearch.autoconfigure + +import org.springframework.boot.autoconfigure.AutoConfigurationImportFilter +import org.springframework.boot.autoconfigure.AutoConfigurationMetadata + +/** + * Vetoes Spring Boot's built-in Elasticsearch auto-configurations. + * + * The `spring-data-opensearch-starter` transitively puts `spring-data-elasticsearch` (and its + * Elasticsearch client classes) on the classpath. Spring Boot detects those classes and activates + * its own Elasticsearch auto-configuration — including a reactive REST client configuration that + * fails to construct against OpenSearch and aborts application startup with + * "Lookup method resolution failed". + * + * OpenSearch connectivity is provided instead by spring-data-opensearch's own auto-configuration + * (driven by the `opensearch.*` properties), so Boot's Elasticsearch auto-configs are not just + * unnecessary but actively harmful. Excluding them here — inside the library — means any consuming + * application gets a working setup out of the box, without having to add + * `spring.autoconfigure.exclude` entries to its own configuration. + */ +class ExcludeElasticsearchAutoConfigurationFilter : AutoConfigurationImportFilter { + + override fun match( + autoConfigurationClasses: Array, + autoConfigurationMetadata: AutoConfigurationMetadata, + ): BooleanArray = BooleanArray(autoConfigurationClasses.size) { index -> + autoConfigurationClasses[index] !in EXCLUDED_AUTO_CONFIGURATIONS + } + + companion object { + private val EXCLUDED_AUTO_CONFIGURATIONS = setOf( + "org.springframework.boot.autoconfigure.elasticsearch.ElasticsearchClientAutoConfiguration", + "org.springframework.boot.autoconfigure.elasticsearch.ElasticsearchRestClientAutoConfiguration", + "org.springframework.boot.autoconfigure.elasticsearch.ReactiveElasticsearchClientAutoConfiguration", + "org.springframework.boot.autoconfigure.data.elasticsearch.ElasticsearchDataAutoConfiguration", + "org.springframework.boot.autoconfigure.data.elasticsearch.ElasticsearchRepositoriesAutoConfiguration", + "org.springframework.boot.autoconfigure.data.elasticsearch.ReactiveElasticsearchRepositoriesAutoConfiguration", + ) + } +} diff --git a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/domain/JsonSchemaDocumentOsDocument.kt b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/domain/JsonSchemaDocumentOsDocument.kt new file mode 100644 index 0000000000..c1fac61acf --- /dev/null +++ b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/domain/JsonSchemaDocumentOsDocument.kt @@ -0,0 +1,85 @@ +/* + * Copyright 2015-2026 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.ritense.document.opensearch.domain + +import org.springframework.data.annotation.Id +import org.springframework.data.annotation.Version +import org.springframework.data.elasticsearch.annotations.Document +import org.springframework.data.elasticsearch.annotations.Field +import org.springframework.data.elasticsearch.annotations.FieldType +import org.springframework.data.elasticsearch.annotations.InnerField +import org.springframework.data.elasticsearch.annotations.MultiField +import java.time.LocalDateTime + +/** + * OpenSearch read model for [com.ritense.document.domain.impl.JsonSchemaDocument]. + * + * Uses [Map] types for dynamic content and typed classes for known structure fields. + * [definitionId] uses [OsDefinitionId] / [OsBlueprintId] so sub-fields are mapped as + * [FieldType.Keyword] directly — avoids unnecessary text analysis and removes the need + * for `.keyword` suffix in term queries. + * + * The [contentText] field holds space-separated leaf values from [content] and is indexed + * as both [FieldType.Text] (for analyzed search) and [FieldType.Keyword] (for wildcard search + * preserving partial-match behaviour for wildcard queries). + */ +@Document(indexName = "json_schema_document", createIndex = false) +data class JsonSchemaDocumentOsDocument( + @Id val id: String, + @Field(type = FieldType.Object) val content: Map?, + @Field(type = FieldType.Object) val definitionId: OsDefinitionId?, + @Field(type = FieldType.Date, format = [], pattern = ["uuuu-MM-dd'T'HH:mm:ss.SSS"]) val createdOn: LocalDateTime?, + @Field(type = FieldType.Date, format = [], pattern = ["uuuu-MM-dd'T'HH:mm:ss.SSS"]) val modifiedOn: LocalDateTime?, + @Field(type = FieldType.Keyword) val createdBy: String?, + @Field(type = FieldType.Long) val sequence: Long?, + @Field(type = FieldType.Integer) val version: Int?, + @Field(type = FieldType.Keyword) val assigneeId: String?, + @Field(type = FieldType.Keyword) val assigneeFullName: String?, + @Field(type = FieldType.Keyword) val internalStatus: String?, + @Field(type = FieldType.Object) val caseTags: List?, + @Field(type = FieldType.Object, enabled = false) val relations: Any?, + @Field(type = FieldType.Object, enabled = false) val relatedFiles: Any?, + @Field(type = FieldType.Date, format = [], pattern = ["uuuu-MM-dd'T'HH:mm:ss.SSS"]) val retentionDate: LocalDateTime?, + @MultiField( + mainField = Field(type = FieldType.Text), + otherFields = [InnerField(suffix = "keyword", type = FieldType.Keyword)], + ) + val contentText: String? = null, + // OpenSearch external version (maps to _version metadata, VersionType.EXTERNAL — not a source field, so + // no index-mapping change). Populated from the JPA optimistic-lock counter; "highest version wins" makes + // redundant reconciler re-sends and stale async writes benign version-conflict no-ops. + @Version val indexVersion: Long? = null, +) + +data class OsDefinitionId( + @Field(type = FieldType.Keyword) val name: String?, + @Field(type = FieldType.Long) val version: Long?, + @Field(type = FieldType.Object) val blueprintId: OsBlueprintId?, +) + +data class OsBlueprintId( + @Field(type = FieldType.Keyword) val blueprintType: String?, + @Field(type = FieldType.Keyword) val blueprintKey: String?, + @Field(type = FieldType.Keyword) val blueprintVersionTag: String?, + @Field(type = FieldType.Boolean) val isBuildingBlock: Boolean?, + @Field(type = FieldType.Boolean) val isCase: Boolean?, +) + +data class OsCaseTag( + @Field(type = FieldType.Keyword) val key: String?, + @Field(type = FieldType.Keyword) val name: String?, +) diff --git a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/domain/OpenSearchReconcileState.kt b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/domain/OpenSearchReconcileState.kt new file mode 100644 index 0000000000..18a6edd07d --- /dev/null +++ b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/domain/OpenSearchReconcileState.kt @@ -0,0 +1,46 @@ +/* + * Copyright 2015-2026 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.ritense.document.opensearch.domain + +import jakarta.persistence.Column +import jakarta.persistence.Entity +import jakarta.persistence.Id +import jakarta.persistence.Table +import java.time.LocalDateTime + +/** + * Single-row persisted state for the OpenSearch reconciler: the [watermark] is the highest + * `json_schema_document.changed_on` value that has been fully reconciled into OpenSearch. The next cycle + * scans everything changed after (watermark − overlap). The watermark is advanced only after a completely + * successful cycle, so an OpenSearch outage simply parks it until recovery. + */ +@Entity +@Table(name = "document_index_reconcile_state") +class OpenSearchReconcileState( + + @Id + @Column(name = "id") + val id: String = SINGLETON_ID, + + @Column(name = "watermark", nullable = false) + var watermark: LocalDateTime, +) { + companion object { + /** There is only ever one reconcile-state row; this is its fixed primary key. */ + const val SINGLETON_ID = "SINGLETON" + } +} diff --git a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/domain/OpenSearchReindexRun.kt b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/domain/OpenSearchReindexRun.kt new file mode 100644 index 0000000000..5267309e1d --- /dev/null +++ b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/domain/OpenSearchReindexRun.kt @@ -0,0 +1,145 @@ +/* + * Copyright 2015-2026 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.ritense.document.opensearch.domain + +import jakarta.persistence.Column +import jakarta.persistence.Entity +import jakarta.persistence.EnumType +import jakarta.persistence.Enumerated +import jakarta.persistence.Id +import jakarta.persistence.Table +import java.time.LocalDateTime +import java.util.UUID + +/** + * Persisted record of a single OpenSearch re-index run. + * + * The state lives in the database (rather than in JVM memory) so that: + * - status is consistent regardless of which clustered instance answers a query, + * - progress survives a crash/redeploy ([lastId] is the resume cursor), + * - an orphaned [ReindexRunStatus.RUNNING] row can be reconciled on startup. + * + * [scope] holds the serialized [com.ritense.document.opensearch.service.ReindexRequest] (JSON string) + * for auditing/display only — it is never queried as JSON in the database. + */ +@Entity +@Table(name = "document_opensearch_reindex_run") +class OpenSearchReindexRun( + + @Id + @Column(name = "id") + val id: UUID = UUID.randomUUID(), + + @Enumerated(EnumType.STRING) + @Column(name = "status", nullable = false, length = 20) + var status: ReindexRunStatus = ReindexRunStatus.RUNNING, + + @Column(name = "scope") + val scope: String? = null, + + @Column(name = "page_size", nullable = false) + val pageSize: Int = 0, + + @Column(name = "last_id") + var lastId: UUID? = null, + + @Column(name = "processed_count", nullable = false) + var processedCount: Long = 0, + + @Column(name = "skipped_count", nullable = false) + var skippedCount: Long = 0, + + @Column(name = "pruned_count", nullable = false) + var prunedCount: Long = 0, + + @Column(name = "started_on", nullable = false) + val startedOn: LocalDateTime = LocalDateTime.now(), + + @Column(name = "heartbeat_on", nullable = false) + var heartbeatOn: LocalDateTime = LocalDateTime.now(), + + @Column(name = "finished_on") + var finishedOn: LocalDateTime? = null, + + @Column(name = "error") + var error: String? = null, + + @Column(name = "total_count") + var totalCount: Long? = null, + + @Column(name = "prune_checked_count", nullable = false) + var pruneCheckedCount: Long = 0, + + @Column(name = "prune_total_count") + var pruneTotalCount: Long? = null, + + @Column(name = "pruning_phase", nullable = false) + var pruningPhase: Boolean = false, +) { + + /** Records progress after a committed batch: the keyset cursor, counts and a fresh heartbeat. */ + fun recordProgress(lastId: UUID?, processed: Long, skipped: Long, heartbeat: LocalDateTime) { + this.lastId = lastId + this.processedCount = processed + this.skippedCount = skipped + this.heartbeatOn = heartbeat + } + + fun startPruning(totalOsCount: Long, heartbeat: LocalDateTime) { + this.pruningPhase = true + this.pruneTotalCount = totalOsCount + this.pruneCheckedCount = 0 + this.prunedCount = 0 + this.heartbeatOn = heartbeat + } + + fun recordPruneProgress(checked: Long, pruned: Long, heartbeat: LocalDateTime) { + this.pruneCheckedCount = checked + this.prunedCount = pruned + this.heartbeatOn = heartbeat + } + + fun complete(now: LocalDateTime) { + this.status = ReindexRunStatus.COMPLETED + this.finishedOn = now + this.heartbeatOn = now + this.pruningPhase = false + } + + fun fail(now: LocalDateTime, error: String?) { + this.status = ReindexRunStatus.FAILED + this.finishedOn = now + this.heartbeatOn = now + this.error = error + this.pruningPhase = false + } + + fun stop(now: LocalDateTime) { + this.status = ReindexRunStatus.STOPPED + this.finishedOn = now + this.heartbeatOn = now + this.pruningPhase = false + } + + /** Re-arms a previously-finished (FAILED/STOPPED) run so it can be resumed from its cursor. */ + fun resume(now: LocalDateTime) { + this.status = ReindexRunStatus.RUNNING + this.finishedOn = null + this.error = null + this.heartbeatOn = now + } +} diff --git a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/domain/PendingIndexDeletion.kt b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/domain/PendingIndexDeletion.kt new file mode 100644 index 0000000000..b05402bbf9 --- /dev/null +++ b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/domain/PendingIndexDeletion.kt @@ -0,0 +1,45 @@ +/* + * Copyright 2015-2026 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.ritense.document.opensearch.domain + +import jakarta.persistence.Column +import jakarta.persistence.Entity +import jakarta.persistence.Id +import jakarta.persistence.Table +import java.time.LocalDateTime +import java.util.UUID + +/** + * Durable record that a document was deleted from PostgreSQL and its OpenSearch entry still has to be + * removed. + * + * Written **inside** the deleting transaction (see + * [com.ritense.document.opensearch.handler.PendingIndexDeletionListener]) so it commits atomically with + * the delete — surviving an OpenSearch outage of any length. The reconciler drains these rows at + * O(deletes): remove each id from the index, then delete the drained rows. Idempotent. + */ +@Entity +@Table(name = "document_index_pending_deletion") +class PendingIndexDeletion( + + @Id + @Column(name = "document_id") + val documentId: UUID, + + @Column(name = "deleted_on", nullable = false) + val deletedOn: LocalDateTime = LocalDateTime.now(), +) diff --git a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/domain/ReindexRunStatus.kt b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/domain/ReindexRunStatus.kt new file mode 100644 index 0000000000..3c7239c762 --- /dev/null +++ b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/domain/ReindexRunStatus.kt @@ -0,0 +1,31 @@ +/* + * Copyright 2015-2026 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.ritense.document.opensearch.domain + +enum class ReindexRunStatus { + /** The run is currently in progress (or was, for a row left behind by a crashed instance). */ + RUNNING, + + /** The run finished and indexed all documents in scope. */ + COMPLETED, + + /** The run aborted with an error; resumable from [OpenSearchReindexRun.lastId]. */ + FAILED, + + /** The run was cancelled (e.g. graceful shutdown); resumable from [OpenSearchReindexRun.lastId]. */ + STOPPED, +} diff --git a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/handler/DocumentOpenSearchEventListener.kt b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/handler/DocumentOpenSearchEventListener.kt new file mode 100644 index 0000000000..9fe8a5c388 --- /dev/null +++ b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/handler/DocumentOpenSearchEventListener.kt @@ -0,0 +1,131 @@ +/* + * Copyright 2015-2026 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.ritense.document.opensearch.handler + +import com.ritense.document.domain.impl.event.JsonSchemaDocumentCreatedEvent +import com.ritense.document.domain.impl.event.JsonSchemaDocumentModifiedEvent +import com.ritense.document.event.DocumentAssigneeChangedEvent +import com.ritense.document.event.DocumentRetentionPeriodSetEvent +import com.ritense.document.event.DocumentRetentionPeriodUnsetEvent +import com.ritense.document.event.DocumentUnassignedEvent +import com.ritense.document.opensearch.service.DocumentOpenSearchSyncService +import com.ritense.document.opensearch.service.SearchEngineToggle +import com.ritense.valtimo.contract.document.event.DocumentRelatedFileAddedEvent +import com.ritense.valtimo.contract.document.event.DocumentRelatedFileRemovedEvent +import com.ritense.valtimo.contract.event.DocumentDeletedEvent +import io.github.oshai.kotlinlogging.KotlinLogging +import org.springframework.beans.factory.DisposableBean +import org.springframework.transaction.event.TransactionPhase.AFTER_COMMIT +import org.springframework.transaction.event.TransactionalEventListener +import java.util.UUID +import java.util.concurrent.ExecutorService +import java.util.concurrent.LinkedBlockingQueue +import java.util.concurrent.RejectedExecutionException +import java.util.concurrent.ThreadPoolExecutor +import java.util.concurrent.TimeUnit + +/** + * Best-effort, low-latency live sync of single-document mutations into OpenSearch. + * + * Listens to the real Spring application events emitted inside each document-mutation transaction and, + * **after commit**, reloads the document by id and upserts it (or deletes it) in OpenSearch on a managed + * single-thread daemon executor. Every task is fully isolated: a failure (e.g. OpenSearch unreachable) is + * logged and swallowed so it can never fail or roll back the originating business transaction. Missed + * writes are repaired by [com.ritense.document.opensearch.service.DocumentOpenSearchReconcileService]; + * this listener is only about freshness. + * + * Status/tags changes and bulk deletes have no dedicated live event and are handled by the reconciler + * (upserts) and the pending-index-deletion drain (deletes) respectively. + */ +class DocumentOpenSearchEventListener( + private val syncService: DocumentOpenSearchSyncService, + private val toggle: SearchEngineToggle, +) : DisposableBean { + + private val executor: ExecutorService = ThreadPoolExecutor( + 1, 1, 0L, TimeUnit.MILLISECONDS, + LinkedBlockingQueue(QUEUE_CAPACITY), + { runnable -> Thread(runnable, "opensearch-live-sync").apply { isDaemon = true } }, + ThreadPoolExecutor.DiscardOldestPolicy() + ) + + @TransactionalEventListener(phase = AFTER_COMMIT) + fun onCreated(event: JsonSchemaDocumentCreatedEvent) = enqueueUpsert(event.documentId().id) + + @TransactionalEventListener(phase = AFTER_COMMIT) + fun onModified(event: JsonSchemaDocumentModifiedEvent) = enqueueUpsert(event.documentId().id) + + @TransactionalEventListener(phase = AFTER_COMMIT) + fun onAssigneeChanged(event: DocumentAssigneeChangedEvent) = enqueueUpsert(event.documentId) + + @TransactionalEventListener(phase = AFTER_COMMIT) + fun onUnassigned(event: DocumentUnassignedEvent) = enqueueUpsert(event.documentId) + + @TransactionalEventListener(phase = AFTER_COMMIT) + fun onRetentionSet(event: DocumentRetentionPeriodSetEvent) = enqueueUpsert(event.getDocumentId()) + + @TransactionalEventListener(phase = AFTER_COMMIT) + fun onRetentionUnset(event: DocumentRetentionPeriodUnsetEvent) = enqueueUpsert(event.getDocumentId()) + + @TransactionalEventListener(phase = AFTER_COMMIT) + fun onRelatedFileAdded(event: DocumentRelatedFileAddedEvent) = enqueueUpsert(event.documentId) + + @TransactionalEventListener(phase = AFTER_COMMIT) + fun onRelatedFileRemoved(event: DocumentRelatedFileRemovedEvent) = enqueueUpsert(event.documentId) + + @TransactionalEventListener(phase = AFTER_COMMIT) + fun onDeleted(event: DocumentDeletedEvent) = enqueueDelete(event.caseDocumentId) + + private fun enqueueUpsert(documentId: UUID) = submit { syncService.upsertById(documentId) } + + private fun enqueueDelete(documentId: UUID) = submit { syncService.delete(documentId) } + + private fun submit(task: () -> Unit) { + // Engine off (feature toggled off or OpenSearch disabled): skip the write entirely — no thread, + // no OpenSearch call. The reconciler catches up from its watermark once the engine is re-enabled. + if (!toggle.isOpenSearchActive()) return + try { + executor.execute { + try { + task() + } catch (e: Exception) { + logger.warn(e) { "Live OpenSearch sync failed — the reconciler will repair the index on its next cycle" } + } + } + } catch (e: RejectedExecutionException) { + logger.warn(e) { "Live OpenSearch sync rejected (executor shutting down) — the reconciler will repair the index" } + } + } + + override fun destroy() { + executor.shutdown() + try { + if (!executor.awaitTermination(SHUTDOWN_TIMEOUT_SECONDS, TimeUnit.SECONDS)) { + executor.shutdownNow() + } + } catch (e: InterruptedException) { + executor.shutdownNow() + Thread.currentThread().interrupt() + } + } + + companion object { + private val logger = KotlinLogging.logger {} + private const val SHUTDOWN_TIMEOUT_SECONDS = 30L + private const val QUEUE_CAPACITY = 1000 + } +} diff --git a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/handler/PendingIndexDeletionListener.kt b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/handler/PendingIndexDeletionListener.kt new file mode 100644 index 0000000000..b4ec98a229 --- /dev/null +++ b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/handler/PendingIndexDeletionListener.kt @@ -0,0 +1,48 @@ +/* + * Copyright 2015-2026 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.ritense.document.opensearch.handler + +import com.ritense.document.opensearch.domain.PendingIndexDeletion +import com.ritense.document.opensearch.repository.PendingIndexDeletionRepository +import com.ritense.valtimo.contract.event.DocumentDeletedEvent +import io.github.oshai.kotlinlogging.KotlinLogging +import org.springframework.context.event.EventListener + +/** + * Records a durable pending index deletion for every deleted document. + * + * Deliberately a **synchronous, in-transaction** [EventListener] (not `@TransactionalEventListener`): the + * [DocumentDeletedEvent] is published inside the deleting transaction, so the pending-deletion row commits + * atomically with the delete. This is what guarantees deletes survive an OpenSearch outage — the + * reconciler drains the pending deletion once OpenSearch is reachable again. The best-effort AFTER_COMMIT + * delete in [DocumentOpenSearchEventListener] still runs for freshness; this listener is the durability + * backstop. + */ +open class PendingIndexDeletionListener( + private val pendingIndexDeletionRepository: PendingIndexDeletionRepository, +) { + + @EventListener + open fun onDocumentDeleted(event: DocumentDeletedEvent) { + pendingIndexDeletionRepository.save(PendingIndexDeletion(documentId = event.caseDocumentId)) + logger.debug { "Recorded pending index deletion for document ${event.caseDocumentId}" } + } + + companion object { + private val logger = KotlinLogging.logger {} + } +} diff --git a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/repository/JsonSchemaDocumentOpenSearchRepository.kt b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/repository/JsonSchemaDocumentOpenSearchRepository.kt new file mode 100644 index 0000000000..2241604807 --- /dev/null +++ b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/repository/JsonSchemaDocumentOpenSearchRepository.kt @@ -0,0 +1,22 @@ +/* + * Copyright 2015-2024 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.ritense.document.opensearch.repository + +import com.ritense.document.opensearch.domain.JsonSchemaDocumentOsDocument +import org.springframework.data.elasticsearch.repository.ElasticsearchRepository + +interface JsonSchemaDocumentOpenSearchRepository : ElasticsearchRepository diff --git a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/repository/OpenSearchReconcileStateRepository.kt b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/repository/OpenSearchReconcileStateRepository.kt new file mode 100644 index 0000000000..79e1bb4fd3 --- /dev/null +++ b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/repository/OpenSearchReconcileStateRepository.kt @@ -0,0 +1,22 @@ +/* + * Copyright 2015-2026 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.ritense.document.opensearch.repository + +import com.ritense.document.opensearch.domain.OpenSearchReconcileState +import org.springframework.data.jpa.repository.JpaRepository + +interface OpenSearchReconcileStateRepository : JpaRepository diff --git a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/repository/OpenSearchReindexRunRepository.kt b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/repository/OpenSearchReindexRunRepository.kt new file mode 100644 index 0000000000..48e6c7612b --- /dev/null +++ b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/repository/OpenSearchReindexRunRepository.kt @@ -0,0 +1,33 @@ +/* + * Copyright 2015-2026 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.ritense.document.opensearch.repository + +import com.ritense.document.opensearch.domain.OpenSearchReindexRun +import com.ritense.document.opensearch.domain.ReindexRunStatus +import org.springframework.data.domain.Page +import org.springframework.data.domain.Pageable +import org.springframework.data.jpa.repository.JpaRepository +import java.time.LocalDateTime +import java.util.UUID + +interface OpenSearchReindexRunRepository : JpaRepository { + fun findFirstByOrderByStartedOnDesc(): OpenSearchReindexRun? + fun findFirstByStatusOrderByStartedOnDesc(status: ReindexRunStatus): OpenSearchReindexRun? + fun findAllByStatusAndHeartbeatOnBefore(status: ReindexRunStatus, heartbeatOn: LocalDateTime): List + fun existsByStatusAndHeartbeatOnAfter(status: ReindexRunStatus, heartbeatOn: LocalDateTime): Boolean + fun findAllByOrderByStartedOnDesc(pageable: Pageable): Page +} diff --git a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/repository/PendingIndexDeletionRepository.kt b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/repository/PendingIndexDeletionRepository.kt new file mode 100644 index 0000000000..581a4e801d --- /dev/null +++ b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/repository/PendingIndexDeletionRepository.kt @@ -0,0 +1,26 @@ +/* + * Copyright 2015-2026 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.ritense.document.opensearch.repository + +import com.ritense.document.opensearch.domain.PendingIndexDeletion +import org.springframework.data.domain.Pageable +import org.springframework.data.jpa.repository.JpaRepository +import java.util.UUID + +interface PendingIndexDeletionRepository : JpaRepository { + fun findByOrderByDeletedOnAsc(pageable: Pageable): List +} diff --git a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/security/DocumentOpenSearchHttpSecurityConfigurer.kt b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/security/DocumentOpenSearchHttpSecurityConfigurer.kt new file mode 100644 index 0000000000..d270f8fcf2 --- /dev/null +++ b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/security/DocumentOpenSearchHttpSecurityConfigurer.kt @@ -0,0 +1,50 @@ +/* + * Copyright 2015-2026 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.ritense.document.opensearch.security + +import com.ritense.valtimo.contract.authentication.AuthoritiesConstants.ADMIN +import com.ritense.valtimo.contract.security.config.HttpConfigurerConfigurationException +import com.ritense.valtimo.contract.security.config.HttpSecurityConfigurer +import org.springframework.http.HttpMethod.GET +import org.springframework.http.HttpMethod.POST +import org.springframework.http.HttpMethod.PUT +import org.springframework.security.config.annotation.web.builders.HttpSecurity +import org.springframework.security.web.util.matcher.AntPathRequestMatcher.antMatcher + +class DocumentOpenSearchHttpSecurityConfigurer : HttpSecurityConfigurer { + + override fun configure(http: HttpSecurity) { + try { + http.authorizeHttpRequests { requests -> + requests.requestMatchers(antMatcher(POST, "/api/management/v1/document-opensearch/reindex")) + .hasAuthority(ADMIN) + requests.requestMatchers(antMatcher(GET, "/api/management/v1/document-opensearch/reindex/runs")) + .hasAuthority(ADMIN) + requests.requestMatchers(antMatcher(GET, "/api/management/v1/document-opensearch/reindex/status")) + .hasAuthority(ADMIN) + requests.requestMatchers(antMatcher(GET, "/api/management/v1/document-opensearch/reindex/*")) + .hasAuthority(ADMIN) + requests.requestMatchers(antMatcher(GET, "/api/management/v1/search-engine")) + .hasAuthority(ADMIN) + requests.requestMatchers(antMatcher(PUT, "/api/management/v1/search-engine")) + .hasAuthority(ADMIN) + } + } catch (e: Exception) { + throw HttpConfigurerConfigurationException(e) + } + } +} diff --git a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/ContentTextExtractor.kt b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/ContentTextExtractor.kt new file mode 100644 index 0000000000..067853d855 --- /dev/null +++ b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/ContentTextExtractor.kt @@ -0,0 +1,39 @@ +/* + * Copyright 2015-2024 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.ritense.document.opensearch.service + +import com.fasterxml.jackson.databind.JsonNode + +/** + * Extracts all leaf values from a [JsonNode] as a single space-separated string. + * Used to populate [com.ritense.document.opensearch.domain.JsonSchemaDocumentOsDocument.contentText] + * for full-document search. + */ +fun extractLeafValues(node: JsonNode?): String? { + if (node == null) return null + val parts = mutableListOf() + collectLeaves(node, parts) + return parts.joinToString(" ").ifBlank { null } +} + +private fun collectLeaves(node: JsonNode, out: MutableList) { + when { + node.isObject -> node.fields().forEach { (_, v) -> collectLeaves(v, out) } + node.isArray -> node.forEach { collectLeaves(it, out) } + !node.isNull && !node.isMissingNode -> out.add(node.asText()) + } +} diff --git a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/DelegatingDocumentSearchService.kt b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/DelegatingDocumentSearchService.kt new file mode 100644 index 0000000000..64e11ab693 --- /dev/null +++ b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/DelegatingDocumentSearchService.kt @@ -0,0 +1,105 @@ +/* + * Copyright 2015-2026 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.ritense.document.opensearch.service + +import com.ritense.document.domain.Document +import com.ritense.document.domain.search.AdvancedSearchRequest +import com.ritense.document.domain.search.SearchWithConfigRequest +import com.ritense.document.service.DocumentSearchService +import com.ritense.document.service.impl.SearchRequest +import com.ritense.valtimo.contract.blueprint.BlueprintType +import org.springframework.data.domain.Page +import org.springframework.data.domain.Pageable + +class DelegatingDocumentSearchService( + private val openSearchService: DocumentSearchService, + private val jpaService: DocumentSearchService, + private val toggle: SearchEngineToggle, + private val reindexProgressGate: ReindexProgressGate, +) : DocumentSearchService { + + override fun search( + searchRequest: SearchRequest, + blueprintType: BlueprintType, + pageable: Pageable + ): Page = executeWithFallback { active().search(searchRequest, blueprintType, pageable) } + + override fun search( + documentDefinitionName: String, + blueprintType: BlueprintType, + searchWithConfigRequest: SearchWithConfigRequest, + pageable: Pageable + ): Page = executeWithFallback { + active().search(documentDefinitionName, blueprintType, searchWithConfigRequest, pageable) + } + + override fun search( + documentDefinitionName: String, + blueprintType: BlueprintType, + advancedSearchRequest: AdvancedSearchRequest, + pageable: Pageable + ): Page = executeWithFallback { + active().search(documentDefinitionName, blueprintType, advancedSearchRequest, pageable) + } + + override fun searchForExport( + documentDefinitionName: String, + blueprintType: BlueprintType, + searchWithConfigRequest: SearchWithConfigRequest, + pageable: Pageable + ): Page = executeWithFallback { + active().searchForExport(documentDefinitionName, blueprintType, searchWithConfigRequest, pageable) + } + + override fun count( + documentDefinitionName: String, + blueprintType: BlueprintType, + advancedSearchRequest: AdvancedSearchRequest + ): Long = executeWithFallback { active().count(documentDefinitionName, blueprintType, advancedSearchRequest) } + + private fun active(): DocumentSearchService = + if (toggle.shouldUsePostgres { reindexProgressGate.isReindexInProgress() }) jpaService else openSearchService + + private fun executeWithFallback(block: () -> T): T { + if (toggle.shouldUsePostgres { reindexProgressGate.isReindexInProgress() }) { + return block() + } + return try { + block() + } catch (e: Exception) { + if (isConnectionError(e)) { + toggle.activateFallback() + block() + } else { + throw e + } + } + } + + private fun isConnectionError(e: Exception): Boolean { + val message = e.message?.lowercase() ?: "" + return e is java.net.ConnectException || + e is java.net.SocketTimeoutException || + e is java.net.NoRouteToHostException || + e is java.net.UnknownHostException || + message.contains("connection refused") || + message.contains("connect timed out") || + message.contains("no route to host") || + e.cause?.let { isConnectionError(it as? Exception ?: return false) } ?: false + } + +} diff --git a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchIndexInitializer.kt b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchIndexInitializer.kt new file mode 100644 index 0000000000..85118f708f --- /dev/null +++ b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchIndexInitializer.kt @@ -0,0 +1,59 @@ +/* + * Copyright 2015-2026 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.ritense.document.opensearch.service + +import com.ritense.document.opensearch.domain.JsonSchemaDocumentOsDocument +import io.github.oshai.kotlinlogging.KotlinLogging +import org.springframework.data.elasticsearch.core.ElasticsearchOperations +import org.springframework.data.elasticsearch.core.document.Document + +/** + * Creates the OpenSearch index and mappings if they do not yet exist. Invoked before the engine toggle + * switches to OpenSearch — failure prevents the swap, keeping queries on Postgres until the cluster is healthy. + * [ensureIndex] is idempotent, so repeated calls are safe. + */ +open class DocumentOpenSearchIndexInitializer( + private val elasticsearchOperations: ElasticsearchOperations, +) { + + open fun ensureIndex() { + val indexOps = elasticsearchOperations.indexOps(JsonSchemaDocumentOsDocument::class.java) + if (!indexOps.exists()) { + val settings = Document.create() + settings["index.number_of_replicas"] = 0 + indexOps.create(settings) + + val annotatedMapping = indexOps.createMapping(JsonSchemaDocumentOsDocument::class.java) + val dynamicTemplates = listOf( + mapOf("content_fields_as_text" to mapOf( + "path_match" to "content.*", + "match_mapping_type" to "string", + "mapping" to mapOf( + "type" to "text", + "fields" to mapOf("keyword" to mapOf("type" to "keyword", "ignore_above" to 256)) + ) + )) + ) + annotatedMapping["dynamic_templates"] = dynamicTemplates + indexOps.putMapping(annotatedMapping) + } + } + + companion object { + private val logger = KotlinLogging.logger {} + } +} \ No newline at end of file diff --git a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchQueryService.kt b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchQueryService.kt new file mode 100644 index 0000000000..36aed351c3 --- /dev/null +++ b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchQueryService.kt @@ -0,0 +1,78 @@ +/* + * Copyright 2015-2024 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.ritense.document.opensearch.service + +import com.ritense.authorization.Action +import com.ritense.authorization.AuthorizationService +import com.ritense.document.domain.impl.JsonSchemaDocument +import com.ritense.document.opensearch.authorization.OpenSearchPermissionConditionTranslator +import com.ritense.document.opensearch.authorization.OpenSearchPermissionConditionTranslator.Companion.andAll +import com.ritense.document.opensearch.domain.JsonSchemaDocumentOsDocument +import com.ritense.document.service.JsonSchemaDocumentActionProvider +import com.ritense.valtimo.contract.utils.SecurityUtils +import org.opensearch.index.query.QueryBuilder +import org.opensearch.index.query.QueryBuilders +import org.springframework.data.domain.Page +import org.springframework.data.domain.PageImpl +import org.springframework.data.domain.Pageable +import org.springframework.data.elasticsearch.core.ElasticsearchOperations +import org.springframework.data.elasticsearch.core.query.StringQuery + +class DocumentOpenSearchQueryService( + private val elasticsearchOperations: ElasticsearchOperations, + private val authorizationService: AuthorizationService, + private val translator: OpenSearchPermissionConditionTranslator, +) { + + /** + * Returns a page of documents for the given [definitionName], restricted to those + * the current user is allowed to see (VIEW_LIST action). + */ + fun findAllByDefinitionName(definitionName: String, pageable: Pageable): Page { + val authQuery = buildAuthQuery(JsonSchemaDocumentActionProvider.VIEW_LIST) + val definitionFilter = QueryBuilders.termQuery("definitionId.name", definitionName) + val combined = andAll(listOf(authQuery, definitionFilter)) + + val dataQuery = StringQuery(combined.toString(), pageable) + + val hits = elasticsearchOperations.search(dataQuery, JsonSchemaDocumentOsDocument::class.java) + val total = hits.totalHits + val content = hits.searchHits.mapNotNull { hit -> hit.content } + return PageImpl(content, pageable, total) + } + + /** + * Returns the document with the given [id] if the current user has VIEW permission, + * or `null` if it does not exist or is not accessible. + */ + fun findById(id: String): JsonSchemaDocumentOsDocument? { + val authQuery = buildAuthQuery(JsonSchemaDocumentActionProvider.VIEW) + val idFilter = QueryBuilders.idsQuery().addIds(id) + val combined = andAll(listOf(authQuery, idFilter)) + + val query = StringQuery(combined.toString()) + val hits = elasticsearchOperations.search(query, JsonSchemaDocumentOsDocument::class.java) + return hits.searchHits.firstOrNull()?.content + } + + private fun buildAuthQuery(action: Action): QueryBuilder { + val userRoles = SecurityUtils.getCurrentUserRoles().toSet() + val permissions = authorizationService.getPermissions(JsonSchemaDocument::class.java, action) + .filter { it.role.key in userRoles } + return translator.toQuery(permissions, action) + } +} diff --git a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchReconcileJob.kt b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchReconcileJob.kt new file mode 100644 index 0000000000..68fac30130 --- /dev/null +++ b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchReconcileJob.kt @@ -0,0 +1,48 @@ +/* + * Copyright 2015-2026 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.ritense.document.opensearch.service + +import org.springframework.scheduling.annotation.Scheduled +import java.util.concurrent.atomic.AtomicBoolean + +/** + * Fires the reconcile cycle on a fixed delay. The [running] guard makes overlapping scheduled invocations + * on this node a no-op; cross-node exclusivity is handled by the ShedLock inside the service. `fixedDelay` + * (not `fixedRate`) so a slow cycle never queues up back-to-back runs. + */ +class DocumentOpenSearchReconcileJob( + private val reconcileService: DocumentOpenSearchReconcileService, + private val toggle: SearchEngineToggle, +) { + private val running = AtomicBoolean(false) + + // The PT2M default must match OpenSearchProperties.Reconcile.interval; the live path owns freshness, + // so this safety-net reconciler runs on a relaxed interval. + @Scheduled(fixedDelayString = "\${valtimo.opensearch.reconcile.interval:PT2M}") + fun reconcile() { + // Engine off: skip this cycle without touching OpenSearch. The tick keeps firing cheaply and + // resumes reconciling from the persisted watermark on the first cycle after the engine is re-enabled. + if (!toggle.isOpenSearchActive()) return + if (running.compareAndSet(false, true)) { + try { + reconcileService.reconcile() + } finally { + running.set(false) + } + } + } +} diff --git a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchReconcileService.kt b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchReconcileService.kt new file mode 100644 index 0000000000..d18c27c317 --- /dev/null +++ b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchReconcileService.kt @@ -0,0 +1,223 @@ +/* + * Copyright 2015-2026 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.ritense.document.opensearch.service + +import com.ritense.document.domain.impl.JsonSchemaDocument +import com.ritense.document.opensearch.OpenSearchProperties +import com.ritense.document.opensearch.domain.JsonSchemaDocumentOsDocument +import com.ritense.document.opensearch.domain.OpenSearchReconcileState +import com.ritense.document.opensearch.repository.JsonSchemaDocumentOpenSearchRepository +import com.ritense.document.opensearch.repository.OpenSearchReconcileStateRepository +import com.ritense.document.opensearch.repository.PendingIndexDeletionRepository +import io.github.oshai.kotlinlogging.KotlinLogging +import jakarta.persistence.EntityManager +import net.javacrumbs.shedlock.core.LockConfiguration +import net.javacrumbs.shedlock.core.LockProvider +import org.springframework.data.domain.PageRequest +import org.springframework.transaction.PlatformTransactionManager +import org.springframework.transaction.support.TransactionTemplate +import java.time.Duration +import java.time.Instant +import java.time.LocalDateTime +import java.util.UUID + +/** + * The self-healing backbone: a single-writer, watermark-based incremental reindex that makes the + * OpenSearch index a derived read-model of PostgreSQL. + * + * Each cycle (guarded cluster-wide by a ShedLock named lock so exactly one node runs it): + * 1. reads the persisted watermark (initialised to the current `MAX(changed_on)` on first ever run, so a + * fresh deploy does not re-index the whole corpus — initial population stays with the admin re-index); + * 2. keyset-scans every document with `changed_on > (watermark − overlap)` and idempotently upserts it — + * covering status/tags/anything the live path missed or that happened during an OpenSearch outage; + * 3. drains the pending-index-deletion table (O(deletes), never an O(index) scan); + * 4. advances the watermark to the highest `changed_on` processed — **only** after a fully successful + * cycle. Any failure leaves the watermark parked, so the next cycle simply retries from the same point. + * + * All writes are idempotent, so re-processing the overlap window (or a whole failed cycle) is safe. + */ +open class DocumentOpenSearchReconcileService( + private val entityManager: EntityManager, + private val converter: JsonSchemaDocumentOsConverter, + private val openSearchRepository: JsonSchemaDocumentOpenSearchRepository, + private val stateRepository: OpenSearchReconcileStateRepository, + private val pendingIndexDeletionRepository: PendingIndexDeletionRepository, + private val transactionManager: PlatformTransactionManager, + private val lockProvider: LockProvider, + private val properties: OpenSearchProperties, +) { + + open fun reconcile() { + if (!properties.enabled) return + + val lock = lockProvider.lock( + LockConfiguration(Instant.now(), LOCK_NAME, LOCK_AT_MOST_FOR, Duration.ZERO) + ) + if (lock.isEmpty) { + logger.debug { "Another node is reconciling — skipping this cycle" } + return + } + + try { + val watermark = currentWatermark() + val from = watermark.minus(properties.reconcile.overlap) + val maxSeen = processUpserts(from) + drainPendingDeletions() + if (maxSeen.isAfter(watermark)) { + advanceWatermark(maxSeen) + logger.debug { "Reconcile advanced watermark to $maxSeen" } + } + } catch (e: Exception) { + logger.error(e) { "OpenSearch reconcile cycle failed — watermark not advanced; retrying next cycle" } + } finally { + lock.get().unlock() + } + } + + /** + * Keyset-paginates over `changed_on > from` (tie-broken by id), converting and idempotently upserting + * each page. Returns the highest `changed_on` seen (or [from] when nothing changed). + */ + private fun processUpserts(from: LocalDateTime): LocalDateTime { + var maxSeen = from + var lastChangedOn: LocalDateTime? = null + var lastId: UUID? = null + val pageSize = properties.reconcile.pageSize + val txTemplate = TransactionTemplate(transactionManager).apply { isReadOnly = true } + + while (true) { + val cursorChangedOn = lastChangedOn + val cursorId = lastId + val page = txTemplate.execute { + val batch = fetchPage(from, cursorChangedOn, cursorId, pageSize) + if (batch.isEmpty()) { + null + } else { + val osDocuments = batch.mapNotNull { document -> + try { + converter.toOsDocument(document) + } catch (e: Exception) { + logger.warn(e) { "Failed to convert document ${document.id().id} during reconcile — skipping" } + null + } + } + val last = batch.last() + ReconcilePage(osDocuments, last.changedOn(), last.id().id).also { entityManager.clear() } + } + } ?: break + + page.osDocuments.chunked(JsonSchemaDocumentOsConverter.BULK_CHUNK_SIZE) + .forEach { converter.indexChunk(it) } + + if (page.lastChangedOn.isAfter(maxSeen)) maxSeen = page.lastChangedOn + lastChangedOn = page.lastChangedOn + lastId = page.lastId + } + return maxSeen + } + + /** + * Scoped keyset fetch. Eagerly loads the lazy `internalStatus` `@ManyToOne` so the converted document + * carries the real status key, and keeps a composite `(changed_on, id)` cursor for constant-cost + * pagination that is stable when many rows share the same `changed_on`. + */ + private fun fetchPage( + from: LocalDateTime, + lastChangedOn: LocalDateTime?, + lastId: UUID?, + pageSize: Int, + ): List { + val hasCursor = lastChangedOn != null && lastId != null + val jpql = buildString { + append("SELECT d FROM JsonSchemaDocument d LEFT JOIN FETCH d.internalStatus WHERE d.changedOn > :from") + if (hasCursor) { + append(" AND (d.changedOn > :lastChangedOn OR (d.changedOn = :lastChangedOn AND d.id.id > :lastId))") + } + append(" ORDER BY d.changedOn ASC, d.id.id ASC") + } + val query = entityManager.createQuery(jpql, JsonSchemaDocument::class.java) + query.setParameter("from", from) + if (hasCursor) { + query.setParameter("lastChangedOn", lastChangedOn) + query.setParameter("lastId", lastId) + } + return query.setMaxResults(pageSize).resultList + } + + /** + * Removes pending-deletion documents from OpenSearch in batches, then deletes the drained + * pending-deletion rows. + */ + private fun drainPendingDeletions() { + val batchSize = properties.reconcile.pendingDeletionBatchSize + val readTemplate = TransactionTemplate(transactionManager).apply { isReadOnly = true } + val writeTemplate = TransactionTemplate(transactionManager) + while (true) { + val pendingDeletions = readTemplate.execute { + pendingIndexDeletionRepository.findByOrderByDeletedOnAsc(PageRequest.of(0, batchSize)) + }.orEmpty() + if (pendingDeletions.isEmpty()) break + + pendingDeletions.forEach { openSearchRepository.deleteById(it.documentId.toString()) } + writeTemplate.execute { pendingIndexDeletionRepository.deleteAllById(pendingDeletions.map { it.documentId }) } + logger.debug { "Drained ${pendingDeletions.size} pending index deletion(s) from OpenSearch" } + } + } + + private fun currentWatermark(): LocalDateTime = + requireNotNull( + TransactionTemplate(transactionManager).execute { + stateRepository.findById(OpenSearchReconcileState.SINGLETON_ID) + .map { it.watermark } + .orElseGet { + val initial = initialWatermark() + stateRepository.save(OpenSearchReconcileState(watermark = initial)) + logger.info { "Initialised OpenSearch reconcile watermark to $initial" } + initial + } + } + ) + + private fun initialWatermark(): LocalDateTime = + entityManager + .createQuery("SELECT MAX(d.changedOn) FROM JsonSchemaDocument d", LocalDateTime::class.java) + .singleResult ?: LocalDateTime.now() + + private fun advanceWatermark(newWatermark: LocalDateTime) { + TransactionTemplate(transactionManager).execute { + val state = stateRepository.findById(OpenSearchReconcileState.SINGLETON_ID) + .orElseGet { OpenSearchReconcileState(watermark = newWatermark) } + state.watermark = newWatermark + stateRepository.save(state) + } + } + + private data class ReconcilePage( + val osDocuments: List, + val lastChangedOn: LocalDateTime, + val lastId: UUID, + ) + + companion object { + private val logger = KotlinLogging.logger {} + + const val LOCK_NAME = "document-opensearch-reconcile" + + /** Lock lease per cycle; a cycle should complete well within this. */ + val LOCK_AT_MOST_FOR: Duration = Duration.ofMinutes(10) + } +} diff --git a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchReindexService.kt b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchReindexService.kt new file mode 100644 index 0000000000..9a4ae206f7 --- /dev/null +++ b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchReindexService.kt @@ -0,0 +1,338 @@ +/* + * Copyright 2015-2026 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.ritense.document.opensearch.service + +import com.ritense.document.domain.impl.JsonSchemaDocument +import com.ritense.document.opensearch.domain.JsonSchemaDocumentOsDocument +import com.ritense.document.opensearch.repository.JsonSchemaDocumentOpenSearchRepository +import io.github.oshai.kotlinlogging.KotlinLogging +import jakarta.persistence.EntityManager +import jakarta.persistence.criteria.JoinType +import jakarta.persistence.criteria.Predicate +import net.javacrumbs.shedlock.core.LockConfiguration +import net.javacrumbs.shedlock.core.LockProvider +import org.springframework.beans.factory.DisposableBean +import org.opensearch.index.query.BoolQueryBuilder +import org.opensearch.index.query.QueryBuilders +import org.springframework.data.domain.Page +import org.springframework.data.domain.PageRequest +import org.springframework.data.domain.Pageable +import org.springframework.data.elasticsearch.core.ElasticsearchOperations +import org.opensearch.data.client.orhlc.NativeSearchQueryBuilder +import org.springframework.transaction.PlatformTransactionManager +import org.springframework.transaction.support.TransactionTemplate +import java.time.Duration +import java.time.Instant +import java.time.LocalDateTime +import java.time.format.DateTimeFormatter +import java.util.UUID +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.TimeUnit + +/** + * Re-runnable, scoped re-index of [JsonSchemaDocument] rows into the live `json_schema_document` + * OpenSearch index. + * + * - **Cluster-safe single runner**: a ShedLock named lock ([LOCK_NAME]) guarantees at most one run + * cluster-wide; a concurrent [start] returns `null` (→ HTTP 409). + * - **Persisted, resumable state**: progress (cursor, counts, heartbeat) lives in the database via + * [OpenSearchReindexRunService]; a FAILED/STOPPED run can be resumed from its cursor with an + * idempotent upsert. + * - **Scoped**: only documents matching the [ReindexRequest] filters are (re)indexed; the index stays + * complete and queryable throughout. + * - **Crash-safe refresh**: no global `refresh_interval` toggle — a single explicit refresh runs at + * successful completion. + */ +open class DocumentOpenSearchReindexService( + private val entityManager: EntityManager, + private val converter: JsonSchemaDocumentOsConverter, + private val elasticsearchOperations: ElasticsearchOperations, + private val transactionManager: PlatformTransactionManager, + private val lockProvider: LockProvider, + private val runService: OpenSearchReindexRunService, + private val openSearchRepository: JsonSchemaDocumentOpenSearchRepository, +) : DisposableBean { + + private val executor: ExecutorService = Executors.newSingleThreadExecutor { runnable -> + Thread(runnable, "opensearch-reindex").apply { isDaemon = true } + } + + @Volatile + private var cancelRequested = false + + /** + * Acquires the cluster-wide lock, creates (or resumes) a run record and dispatches the re-index on + * the managed executor. Returns the run id, or `null` if a re-index is already running anywhere in + * the cluster. + */ + fun start(request: ReindexRequest): UUID? { + val lock = lockProvider.lock( + LockConfiguration(Instant.now(), LOCK_NAME, LOCK_AT_MOST_FOR, Duration.ZERO) + ) + if (lock.isEmpty) return null + + cancelRequested = false + val run = try { + runService.startOrResume(request) + } catch (e: Exception) { + lock.get().unlock() + throw e + } + + executor.execute { + try { + reindex(run.id) + } catch (e: Exception) { + logger.error(e) { "Re-index run ${run.id} terminated with error" } + } finally { + lock.get().unlock() + } + } + return run.id + } + + /** + * Runs the chunked, resumable re-index loop for [runId]. + * Each DB page is read in its own short read-only transaction (keeping snapshots short) and the + * persistence context is cleared after every page. Returns the number of documents processed. + */ + open fun reindex(runId: UUID): Long { + val scope = runService.scopeOf(runId) + var lastId: UUID? = runService.cursorOf(runId) + var processed: Long = runService.processedOf(runId) + var skipped = 0L + val pageSize = runService.pageSizeOf(runId) + val txTemplate = TransactionTemplate(transactionManager).apply { isReadOnly = true } + + try { + while (!cancelRequested) { + val cursor = lastId + val batch = txTemplate.execute { + fetchBatch(scope, cursor, pageSize).also { entityManager.clear() } + } ?: break + if (batch.isEmpty()) break + + val docs = batch.mapNotNull { jpaDoc -> + try { + converter.toOsDocument(jpaDoc) + } catch (e: Exception) { + skipped++ + logger.warn(e) { "Failed to convert document — skipping" } + null + } + } + skipped += docs.chunked(JsonSchemaDocumentOsConverter.BULK_CHUNK_SIZE).sumOf { converter.indexChunk(it) } + + processed += docs.size + lastId = batch.last().id().id + runService.recordProgress(runId, lastId, processed, skipped) + } + + if (cancelRequested) { + logger.info { "Re-index run $runId cancelled — marking STOPPED (processed=$processed, skipped=$skipped)" } + runService.stop(runId) + } else { + if (scope.pruneOrphans) { + val pruned = pruneOrphans(runId, scope) + logger.info { "Pruned $pruned orphan document(s) from OpenSearch" } + } + indexOps().refresh() + logger.info { "Re-index run $runId complete (processed=$processed, skipped=$skipped)" } + runService.complete(runId, processed + skipped) + } + } catch (e: Exception) { + runService.fail(runId, e.message) + logger.error(e) { "Re-index failed (run $runId)" } + throw e + } + return processed + } + + /** Status of a specific run (by id) or the most recent run when [runId] is null. */ + fun status(runId: UUID? = null): Map = runService.toStatusMap(runId) + + fun listRuns(pageable: Pageable): Page> = runService.listRuns(pageable) + + /** + * Scoped keyset fetch. Applies the optional [scope] filters, eagerly loads the lazy `internalStatus` + * `@ManyToOne` (C1 — so the detached entity serializes the real status key, not null), and keeps a + * keyset cursor on the primary key for constant-cost pagination. + */ + private fun fetchBatch(scope: ReindexRequest, lastId: UUID?, pageSize: Int): List { + val cb = entityManager.criteriaBuilder + val query = cb.createQuery(JsonSchemaDocument::class.java) + val root = query.from(JsonSchemaDocument::class.java) + root.fetch("internalStatus", JoinType.LEFT) + + val predicates = mutableListOf() + scope.modifiedAfter?.let { predicates += cb.greaterThan(root.get("modifiedOn"), it) } + scope.modifiedBefore?.let { predicates += cb.lessThan(root.get("modifiedOn"), it) } + scope.documentDefinitionName?.let { + predicates += cb.equal(root.get("documentDefinitionId").get("name"), it) + } + scope.documentIds?.takeIf { it.isNotEmpty() }?.let { + predicates += root.get("id").get("id").`in`(it) + } + lastId?.let { predicates += cb.greaterThan(root.get("id").get("id"), it) } + + // Only restrict when there is at least one predicate; an empty where(...) matches no rows. + if (predicates.isNotEmpty()) { + query.where(*predicates.toTypedArray()) + } + query.orderBy(cb.asc(root.get("id").get("id"))) + return entityManager.createQuery(query).setMaxResults(pageSize).resultList + } + + /** + * Scans OpenSearch for documents matching [scope], checks each batch against PostgreSQL, + * and deletes orphans (documents in OpenSearch but not in PostgreSQL). + * Uses scroll API to handle datasets larger than 10k documents. + */ + private fun pruneOrphans(runId: UUID, scope: ReindexRequest): Long { + var pruned = 0L + var checked = 0L + val txTemplate = TransactionTemplate(transactionManager).apply { isReadOnly = true } + + val totalOsCount = countOpenSearchDocs(scope) + runService.startPruning(runId, totalOsCount) + + scrollOpenSearchIds(scope, PRUNE_BATCH_SIZE) { osIds -> + if (cancelRequested) return@scrollOpenSearchIds false + + val existingIds = txTemplate.execute { + findExistingIds(osIds.map { UUID.fromString(it) }) + }.orEmpty() + + val orphans = osIds.filter { UUID.fromString(it) !in existingIds } + orphans.forEach { openSearchRepository.deleteById(it) } + pruned += orphans.size + checked += osIds.size + + runService.recordPruneProgress(runId, checked, pruned) + true + } + return pruned + } + + private fun countOpenSearchDocs(scope: ReindexRequest): Long { + val boolQuery = BoolQueryBuilder() + + scope.documentDefinitionName?.let { + boolQuery.filter(QueryBuilders.termQuery("definitionId.name", it)) + } + scope.modifiedAfter?.let { + boolQuery.filter(QueryBuilders.rangeQuery("modifiedOn").gt(it.format(OS_DATE_FORMAT))) + } + scope.modifiedBefore?.let { + boolQuery.filter(QueryBuilders.rangeQuery("modifiedOn").lt(it.format(OS_DATE_FORMAT))) + } + scope.documentIds?.takeIf { it.isNotEmpty() }?.let { ids -> + boolQuery.filter(QueryBuilders.idsQuery().addIds(*ids.map { it.toString() }.toTypedArray())) + } + + val query = NativeSearchQueryBuilder() + .withQuery(boolQuery) + .build() + + return elasticsearchOperations.count(query, JsonSchemaDocumentOsDocument::class.java) + } + + /** + * Scrolls through all OpenSearch documents matching [scope], invoking [handler] for each batch. + * Handler returns `true` to continue, `false` to stop early. + */ + private fun scrollOpenSearchIds(scope: ReindexRequest, batchSize: Int, handler: (List) -> Boolean) { + val boolQuery = BoolQueryBuilder() + + scope.documentDefinitionName?.let { + boolQuery.filter(QueryBuilders.termQuery("definitionId.name", it)) + } + scope.modifiedAfter?.let { + boolQuery.filter(QueryBuilders.rangeQuery("modifiedOn").gt(it.format(OS_DATE_FORMAT))) + } + scope.modifiedBefore?.let { + boolQuery.filter(QueryBuilders.rangeQuery("modifiedOn").lt(it.format(OS_DATE_FORMAT))) + } + scope.documentIds?.takeIf { it.isNotEmpty() }?.let { ids -> + boolQuery.filter(QueryBuilders.idsQuery().addIds(*ids.map { it.toString() }.toTypedArray())) + } + + val query = NativeSearchQueryBuilder() + .withQuery(boolQuery) + .withPageable(PageRequest.of(0, batchSize)) + .build() + + elasticsearchOperations.searchForStream(query, JsonSchemaDocumentOsDocument::class.java).use { stream -> + val iterator = stream.iterator() + val batch = mutableListOf() + + while (iterator.hasNext()) { + val id = iterator.next().id ?: continue + batch.add(id) + if (batch.size >= batchSize) { + if (!handler(batch.toList())) return + batch.clear() + } + } + if (batch.isNotEmpty()) { + handler(batch) + } + } + } + + private fun findExistingIds(ids: List): Set { + if (ids.isEmpty()) return emptySet() + return entityManager.createQuery( + "SELECT d.id.id FROM JsonSchemaDocument d WHERE d.id.id IN :ids", + UUID::class.java + ).setParameter("ids", ids).resultList.toSet() + } + + private fun indexOps() = elasticsearchOperations.indexOps(JsonSchemaDocumentOsDocument::class.java) + + /** Signals an in-flight run to stop gracefully and shuts the executor down on context close. */ + override fun destroy() { + cancelRequested = true + executor.shutdown() + try { + if (!executor.awaitTermination(SHUTDOWN_TIMEOUT_SECONDS, TimeUnit.SECONDS)) { + executor.shutdownNow() + } + } catch (e: InterruptedException) { + executor.shutdownNow() + Thread.currentThread().interrupt() + } + } + + companion object { + private val logger = KotlinLogging.logger {} + private val OS_DATE_FORMAT: DateTimeFormatter = DateTimeFormatter.ofPattern("uuuu-MM-dd'T'HH:mm:ss.SSS") + + const val LOCK_NAME = "document-opensearch-reindex" + + /** + * Generous lock lease. The persisted run-state heartbeat is the robust liveness signal; the lock + * is only the cluster-wide mutex. A run exceeding this could in theory let a second runner start — + * acceptable because all writes are idempotent upserts. + */ + val LOCK_AT_MOST_FOR: Duration = Duration.ofHours(6) + + private const val SHUTDOWN_TIMEOUT_SECONDS = 30L + private const val PRUNE_BATCH_SIZE = 1000 + } +} diff --git a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchSyncService.kt b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchSyncService.kt new file mode 100644 index 0000000000..1e0fedc763 --- /dev/null +++ b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchSyncService.kt @@ -0,0 +1,78 @@ +/* + * Copyright 2015-2026 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.ritense.document.opensearch.service + +import com.ritense.authorization.AuthorizationContext +import com.ritense.document.domain.impl.JsonSchemaDocumentId +import com.ritense.document.opensearch.repository.JsonSchemaDocumentOpenSearchRepository +import com.ritense.document.repository.impl.JsonSchemaDocumentRepository +import io.github.oshai.kotlinlogging.KotlinLogging +import org.springframework.data.elasticsearch.VersionConflictException +import org.springframework.transaction.PlatformTransactionManager +import org.springframework.transaction.support.TransactionTemplate +import java.util.UUID + +/** + * Reloads the current state of a document from PostgreSQL (the source of truth) and mirrors it into + * OpenSearch. Callers pass only the document id; the document is (re)read here so a coalesced/late write + * always reflects the latest committed state rather than a stale event payload. + * + * This service does not swallow OpenSearch failures — the caller (the best-effort live listener) is + * responsible for isolating them. Any missed or failed write is repaired by the reconciler. + */ +open class DocumentOpenSearchSyncService( + private val repository: JsonSchemaDocumentOpenSearchRepository, + private val documentRepository: JsonSchemaDocumentRepository, + private val converter: JsonSchemaDocumentOsConverter, + transactionManager: PlatformTransactionManager, +) { + + // Read-only transaction so the lazy associations touched during conversion can be initialised. + private val readOnlyTransactionTemplate = TransactionTemplate(transactionManager).apply { isReadOnly = true } + + /** + * Reloads [documentId] and upserts it into OpenSearch. A document that no longer exists (already + * deleted) is skipped — its removal is handled by [delete] / the pending-index-deletion drain. + */ + open fun upsertById(documentId: UUID) { + val osDocument = readOnlyTransactionTemplate.execute { + AuthorizationContext.runWithoutAuthorization { + documentRepository.findById(JsonSchemaDocumentId.existingId(documentId)).orElse(null) + }?.let { converter.toOsDocument(it) } + } + if (osDocument == null) { + logger.debug { "Document $documentId not found on reload — skipping upsert (likely deleted)" } + return + } + try { + repository.save(osDocument) + logger.debug { "Upserted document $documentId in OpenSearch" } + } catch (e: VersionConflictException) { + // A newer/equal version is already indexed (the reconciler or a later event won the race). + logger.debug { "Document $documentId already at newer version in OpenSearch — skipping live upsert" } + } + } + + open fun delete(documentId: UUID) { + repository.deleteById(documentId.toString()) + logger.debug { "Deleted document $documentId from OpenSearch" } + } + + companion object { + private val logger = KotlinLogging.logger {} + } +} diff --git a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/JsonSchemaDocumentOpenSearchService.kt b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/JsonSchemaDocumentOpenSearchService.kt new file mode 100644 index 0000000000..d7adab5499 --- /dev/null +++ b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/JsonSchemaDocumentOpenSearchService.kt @@ -0,0 +1,648 @@ +/* + * Copyright 2015-2026 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.ritense.document.opensearch.service + +import com.fasterxml.jackson.databind.ObjectMapper +import com.fasterxml.jackson.databind.node.ArrayNode +import com.ritense.authorization.Action +import com.ritense.authorization.AuthorizationContext.Companion.runWithoutAuthorization +import com.ritense.authorization.AuthorizationService +import com.ritense.case.service.CaseDefinitionService +import com.ritense.document.domain.impl.JsonSchemaDocument +import com.ritense.document.domain.impl.JsonSchemaDocumentId +import com.ritense.document.domain.search.AdvancedSearchRequest +import com.ritense.document.domain.search.AssigneeFilter +import com.ritense.document.domain.search.DatabaseSearchType +import com.ritense.document.domain.search.SearchOperator +import com.ritense.document.domain.search.SearchRequestMapper +import com.ritense.document.domain.search.SearchRequestValidator +import com.ritense.document.domain.search.SearchWithConfigRequest +import com.ritense.document.event.DocumentsListed +import com.ritense.document.opensearch.authorization.OpenSearchPermissionConditionTranslator +import com.ritense.document.opensearch.authorization.OpenSearchPermissionConditionTranslator.Companion.andAll +import com.ritense.document.opensearch.domain.JsonSchemaDocumentOsDocument +import com.ritense.document.repository.impl.JsonSchemaDocumentRepository +import com.ritense.document.service.DocumentSearchService +import com.ritense.document.service.JsonSchemaDocumentActionProvider +import com.ritense.document.service.SearchFieldService +import com.ritense.document.service.impl.SearchRequest +import com.ritense.outbox.OutboxService +import com.ritense.valtimo.contract.authentication.UserManagementService +import com.ritense.valtimo.contract.blueprint.BlueprintType +import com.ritense.valtimo.contract.utils.RequestHelper +import com.ritense.document.domain.impl.searchfield.SearchField +import com.ritense.document.domain.impl.searchfield.SearchFieldDataType +import com.ritense.document.domain.impl.searchfield.SearchFieldMatchType +import com.ritense.valtimo.contract.utils.SecurityUtils +import org.apache.commons.lang3.NotImplementedException +import org.opensearch.index.query.Operator +import org.opensearch.index.query.QueryBuilder +import org.opensearch.index.query.QueryBuilders +import org.springframework.data.domain.Page +import org.springframework.data.domain.PageImpl +import org.springframework.data.domain.PageRequest +import org.springframework.data.domain.Pageable +import org.springframework.data.domain.Sort +import org.springframework.data.elasticsearch.core.ElasticsearchOperations +import org.springframework.data.elasticsearch.core.query.StringQuery +import java.time.Instant +import java.time.LocalDate +import java.time.ZoneOffset +import java.time.format.DateTimeFormatter +import org.springframework.transaction.annotation.Transactional + +@Transactional +class JsonSchemaDocumentOpenSearchService( + private val elasticsearchOperations: ElasticsearchOperations, + private val translator: OpenSearchPermissionConditionTranslator, + private val authorizationService: AuthorizationService, + private val jpaRepository: JsonSchemaDocumentRepository, + private val userManagementService: UserManagementService, + private val searchFieldService: SearchFieldService, + private val outboxService: OutboxService, + private val objectMapper: ObjectMapper, + private val caseDefinitionService: CaseDefinitionService, +) : DocumentSearchService { + + override fun search( + searchRequest: SearchRequest, + blueprintType: BlueprintType, + pageable: Pageable + ): Page { + val parts = mutableListOf() + + parts.add(buildAuthQuery(JsonSchemaDocumentActionProvider.VIEW_LIST)) + parts.add(QueryBuilders.termQuery(BLUEPRINT_TYPE_FIELD, blueprintType.name)) + + if (!searchRequest.documentDefinitionName.isNullOrEmpty()) { + parts.add(QueryBuilders.termQuery(DEFINITION_NAME_FIELD, searchRequest.documentDefinitionName)) + } + if (!searchRequest.createdBy.isNullOrEmpty()) { + parts.add(QueryBuilders.termQuery("createdBy", searchRequest.createdBy)) + } + if (searchRequest.sequence != null) { + parts.add(QueryBuilders.termQuery("sequence", searchRequest.sequence)) + } + val globalFilter = searchRequest.globalSearchFilter?.takeIf { it.isNotEmpty() } + if (globalFilter != null) { + if (!searchRequest.documentDefinitionName.isNullOrEmpty()) { + val searchFields = runWithoutAuthorization { + searchFieldService.getSearchFields(searchRequest.documentDefinitionName) + } + if (searchFields.isNotEmpty()) { + parts.add(buildGlobalSearchQuery(globalFilter.trim(), searchFields)) + } else { + parts.add(MATCH_NONE) + } + } else { + val scopedQuery = buildGlobalSearchQueryForAllDefinitions(globalFilter.trim()) + parts.add(scopedQuery) + } + } + searchRequest.otherFilters?.forEach { sc -> + parts.add(QueryBuilders.termQuery("content.${sc.path}", sc.value)) + } + + return executeSearch(andAll(parts), pageable) + } + + override fun search( + documentDefinitionName: String, + blueprintType: BlueprintType, + searchWithConfigRequest: SearchWithConfigRequest, + pageable: Pageable + ): Page { + val zoneOffset = RequestHelper.getZoneOffset() + val searchFieldMap = searchFieldService.getSearchFields(documentDefinitionName) + .associateBy { it.key } + + val otherFilters = searchWithConfigRequest.otherFilters + .map { filter -> SearchRequestMapper.toOtherFilter(filter, searchFieldMap[filter.key], zoneOffset) } + + val advancedSearchRequest = SearchRequestMapper.toAdvancedSearchRequest(searchWithConfigRequest, otherFilters) + + return search( + documentDefinitionName, + blueprintType, + advancedSearchRequest, + pageable, + JsonSchemaDocumentActionProvider.VIEW_LIST + ) + } + + override fun search( + documentDefinitionName: String, + blueprintType: BlueprintType, + advancedSearchRequest: AdvancedSearchRequest, + pageable: Pageable + ): Page { + return search( + documentDefinitionName, + blueprintType, + advancedSearchRequest, + pageable, + JsonSchemaDocumentActionProvider.VIEW_LIST + ) + } + + override fun searchForExport( + documentDefinitionName: String, + blueprintType: BlueprintType, + searchWithConfigRequest: SearchWithConfigRequest, + pageable: Pageable + ): Page { + val zoneOffset = RequestHelper.getZoneOffset() + val searchFieldMap = searchFieldService.getSearchFields(documentDefinitionName) + .associateBy { it.key } + + val otherFilters = searchWithConfigRequest.otherFilters + .map { filter -> SearchRequestMapper.toOtherFilter(filter, searchFieldMap[filter.key], zoneOffset) } + + val advancedSearchRequest = SearchRequestMapper.toAdvancedSearchRequest(searchWithConfigRequest, otherFilters) + + return search( + documentDefinitionName, + blueprintType, + advancedSearchRequest, + pageable, + JsonSchemaDocumentActionProvider.EXPORT + ) + } + + override fun count( + documentDefinitionName: String, + blueprintType: BlueprintType, + advancedSearchRequest: AdvancedSearchRequest + ): Long { + SearchRequestValidator.validate(advancedSearchRequest) + val combinedQuery = buildCombinedQuery( + documentDefinitionName, + blueprintType, + advancedSearchRequest, + JsonSchemaDocumentActionProvider.VIEW_LIST + ) + val countQuery = StringQuery(combinedQuery.toString()) + return elasticsearchOperations.count(countQuery, JsonSchemaDocumentOsDocument::class.java) + } + + private fun search( + documentDefinitionName: String, + blueprintType: BlueprintType, + advancedSearchRequest: AdvancedSearchRequest, + pageable: Pageable, + action: Action + ): Page { + SearchRequestValidator.validate(advancedSearchRequest) + val combinedQuery = buildCombinedQuery(documentDefinitionName, blueprintType, advancedSearchRequest, action) + return executeSearch(combinedQuery, pageable) + } + + private fun buildCombinedQuery( + documentDefinitionName: String?, + blueprintType: BlueprintType, + searchRequest: AdvancedSearchRequest, + action: Action + ): QueryBuilder { + val parts = mutableListOf() + + parts.add(buildAuthQuery(action)) + parts.add(QueryBuilders.termQuery(BLUEPRINT_TYPE_FIELD, blueprintType.name)) + + if (!documentDefinitionName.isNullOrEmpty()) { + parts.add(QueryBuilders.termQuery(DEFINITION_NAME_FIELD, documentDefinitionName)) + } + + if (searchRequest.assigneeFilter != null && searchRequest.assigneeFilter != AssigneeFilter.ALL) { + parts.add(buildAssigneeFilterQuery(searchRequest.assigneeFilter)) + } + + if (!searchRequest.statusFilter.isNullOrEmpty()) { + parts.add(buildStatusFilterQuery(searchRequest.statusFilter)) + } + + if (!searchRequest.caseTagsFilter.isNullOrEmpty()) { + parts.add(QueryBuilders.termsQuery("caseTags.key", searchRequest.caseTagsFilter.toList())) + } + + if (!searchRequest.otherFilters.isNullOrEmpty()) { + parts.add(buildOtherFiltersQuery(searchRequest.otherFilters, searchRequest.searchOperator)) + } + + val globalFilter = searchRequest.globalSearchFilter?.takeIf { it.isNotEmpty() } + if (globalFilter != null) { + if (!documentDefinitionName.isNullOrEmpty()) { + val searchFields = runWithoutAuthorization { + searchFieldService.getSearchFields(documentDefinitionName) + } + if (searchFields.isNotEmpty()) { + parts.add(buildGlobalSearchQuery(globalFilter.trim(), searchFields)) + } else { + parts.add(MATCH_NONE) + } + } else { + val scopedQuery = buildGlobalSearchQueryForAllDefinitions(globalFilter.trim()) + parts.add(scopedQuery) + } + } + + return andAll(parts) + } + + private fun buildAuthQuery(action: Action): QueryBuilder { + val userRoles = SecurityUtils.getCurrentUserRoles().toSet() + val permissions = authorizationService.getPermissions(JsonSchemaDocument::class.java, action) + .filter { it.role.key in userRoles } + return translator.toQuery(permissions, action) + } + + private fun buildAssigneeFilterQuery(filter: AssigneeFilter): QueryBuilder { + val userId = userManagementService.currentUser.username + return when (filter) { + AssigneeFilter.MINE -> QueryBuilders.termQuery("assigneeId", userId) + AssigneeFilter.OPEN -> QueryBuilders.boolQuery().mustNot(QueryBuilders.existsQuery("assigneeId")) + else -> QueryBuilders.matchAllQuery() + } + } + + private fun buildStatusFilterQuery(statusKeys: Set): QueryBuilder { + val conditions = statusKeys.map { key -> + if (key.isNullOrEmpty()) { + QueryBuilders.boolQuery().mustNot(QueryBuilders.existsQuery("internalStatus")) + } else { + QueryBuilders.termQuery("internalStatus", key) + } + } + return if (conditions.size == 1) conditions.first() + else QueryBuilders.boolQuery().apply { + conditions.forEach { should(it) } + minimumShouldMatch(1) + } + } + + private fun buildOtherFiltersQuery( + filters: List, + operator: SearchOperator? + ): QueryBuilder { + val filterQueries = filters.map { buildSingleFilterQuery(it) } + return if (operator == SearchOperator.OR) { + QueryBuilders.boolQuery().apply { + filterQueries.forEach { should(it) } + minimumShouldMatch(1) + } + } else { + andAll(filterQueries) + } + } + + private fun buildSingleFilterQuery(filter: AdvancedSearchRequest.OtherFilter): QueryBuilder { + val isDocField = filter.path.startsWith(DOC_PREFIX) + val baseField = when { + isDocField -> "content.${filter.path.removePrefix(DOC_PREFIX)}" + filter.path.startsWith(CASE_PREFIX) -> filter.path.removePrefix(CASE_PREFIX) + else -> throw IllegalArgumentException("Search path doesn't start with known prefix: '${filter.path}'") + } + // For doc: fields, string equality/like/in queries should target the .keyword sub-field + val keywordField = if (isDocField) "$baseField.keyword" else baseField + + return when (filter.searchType) { + DatabaseSearchType.EQUAL -> { + val values = filter.getValues() + when { + values.isEmpty() -> QueryBuilders.matchAllQuery() + values.size == 1 -> applyEqualQuery(keywordField, baseField, values[0]) + else -> QueryBuilders.boolQuery().apply { + values.forEach { should(applyEqualQuery(keywordField, baseField, it)) } + minimumShouldMatch(1) + } + } + } + DatabaseSearchType.LIKE -> { + val values = filter.getValues() + when { + values.isEmpty() -> QueryBuilders.matchAllQuery() + values.size == 1 -> applyLikeQuery(keywordField, values[0]) + else -> QueryBuilders.boolQuery().apply { + values.forEach { should(applyLikeQuery(keywordField, it)) } + minimumShouldMatch(1) + } + } + } + DatabaseSearchType.IN -> QueryBuilders.termsQuery(keywordField, filter.getValues()) + DatabaseSearchType.GREATER_THAN_OR_EQUAL_TO -> + QueryBuilders.rangeQuery(baseField).gte(formatInstantForOpenSearch(filter.rangeFromValue()!! as Instant)) + DatabaseSearchType.LESS_THAN_OR_EQUAL_TO -> + QueryBuilders.rangeQuery(baseField).lte(formatInstantForOpenSearch(filter.rangeToValue()!! as Instant)) + DatabaseSearchType.BETWEEN -> + QueryBuilders.rangeQuery(baseField).gte(formatInstantForOpenSearch(filter.rangeFromValue()!! as Instant)).lte(formatInstantForOpenSearch(filter.rangeToValue()!! as Instant)) + else -> throw NotImplementedException("Search type '${filter.searchType}' is not supported in the OpenSearch search service") + } + } + + private fun applyEqualQuery(keywordField: String, baseField: String, value: Any?): QueryBuilder { + return if (value is String) { + // Case-insensitive exact match using term query with caseInsensitive flag + QueryBuilders.termQuery(keywordField, value.trim()).caseInsensitive(true) + } else { + QueryBuilders.termQuery(baseField, value) + } + } + + private fun applyLikeQuery(keywordField: String, value: Any?): QueryBuilder { + if (value !is String) { + throw IllegalArgumentException("LIKE search requires String values, got: ${value?.javaClass?.simpleName}") + } + return QueryBuilders.wildcardQuery(keywordField, "*${value.trim()}*").caseInsensitive(true) + } + + private fun formatInstantForOpenSearch(instant: Instant): String { + return DateTimeFormatter.ofPattern("uuuu-MM-dd'T'HH:mm:ss.SSS") + .withZone(ZoneOffset.UTC) + .format(instant) + } + + private fun parseDateRange(dateValue: String): Pair { + val date = LocalDate.parse(dateValue) + val startOfDay = date.atStartOfDay().atZone(ZoneOffset.UTC).toInstant() + val endOfDay = date.plusDays(1).atStartOfDay().atZone(ZoneOffset.UTC).toInstant() + return Pair( + formatInstantForOpenSearch(startOfDay), + formatInstantForOpenSearch(endOfDay) + ) + } + + private fun executeSearch(combinedQuery: QueryBuilder, pageable: Pageable): Page { + val translatedSort = translateSort(pageable.sort) + val effectivePageable = if (pageable.isPaged) { + PageRequest.of(pageable.pageNumber, pageable.pageSize, translatedSort) + } else { + Pageable.unpaged(translatedSort) + } + + val queryJson = combinedQuery.toString() + val dataQuery = StringQuery(queryJson, effectivePageable) + dataQuery.setTrackTotalHitsUpTo(Int.MAX_VALUE) + + val hits = elasticsearchOperations.search(dataQuery, JsonSchemaDocumentOsDocument::class.java) + val total = hits.totalHits + val ids: List = hits.searchHits.mapNotNull { it.id } + + val docIds = ids.map { JsonSchemaDocumentId.existingId(it) } + val entities = runWithoutAuthorization { jpaRepository.findAllById(docIds) } + val entityMap = entities.associateBy { it.id().toString() } + val orderedEntities = ids.mapNotNull { entityMap[it] } + + outboxService.send { DocumentsListed(objectMapper.valueToTree(orderedEntities)) } + + return PageImpl(orderedEntities, pageable, total) + } + + private fun translateSort(sort: Sort): Sort { + if (sort.isUnsorted) return sort + val orders = sort.map { order -> + val osField = when { + order.property.startsWith(DOC_PREFIX) -> "content.${order.property.removePrefix(DOC_PREFIX)}" + order.property.startsWith(CASE_PREFIX) -> order.property.removePrefix(CASE_PREFIX) + else -> order.property + } + if (order.isAscending) Sort.Order.asc(osField) else Sort.Order.desc(osField) + }.toList() + return Sort.by(orders) + } + + private fun buildGlobalSearchQuery(query: String, searchFields: List): QueryBuilder { + val fieldMap = searchFields.associateBy { removePrefixes(it.path) } + val docFields = searchFields + .filter { it.path?.startsWith(DOC_PREFIX) == true } + .map { "content.${it.path?.removePrefix(DOC_PREFIX)}" } + val caseFields = searchFields + .filter { it.path?.startsWith(CASE_PREFIX) == true } + .filter { it.dataType == SearchFieldDataType.TEXT } + .map { it.path?.removePrefix(CASE_PREFIX) } + + val parsedTerms = parseGlobalSearch(query) + + val unknownFields = parsedTerms + .filter { it.field != null } + .map { removePrefixes(it.field) } + .filter { fieldMap[it] == null } + .distinct() + + if (unknownFields.isNotEmpty()) { + throw IllegalArgumentException( + "Unknown search field(s): ${unknownFields.joinToString(", ")}" + ) + } + + val qualifiedCaseFieldQueries = mutableListOf() + val unqualifiedCaseFieldQueries = mutableListOf() + val queryStringParts = mutableListOf() + + parsedTerms.forEach { term -> + if (term.field != null) { + val fieldPath = removePrefixes(term.field) + val field = fieldMap[fieldPath]!! + val isDocField = field.path?.startsWith(DOC_PREFIX) == true + val osPath = if (isDocField) "content.$fieldPath" else fieldPath + + if (!isDocField) { + if (field.dataType == SearchFieldDataType.DATE || field.dataType == SearchFieldDataType.DATETIME) { + val dateRange = parseDateRange(term.value) + qualifiedCaseFieldQueries.add( + QueryBuilders.rangeQuery(osPath) + .gte(dateRange.first) + .lte(dateRange.second) + ) + } else { + val pattern = if (!term.quoted && field.matchType == SearchFieldMatchType.LIKE) { + "*${term.value}*" + } else { + term.value + } + qualifiedCaseFieldQueries.add( + QueryBuilders.wildcardQuery(osPath, pattern).caseInsensitive(true) + ) + } + } else { + val value = escapeQueryStringValue(term.value) + val wrappedValue = if (!term.quoted && field.matchType == SearchFieldMatchType.LIKE) { + "*$value*" + } else if (term.quoted) { + "\"$value\"" + } else { + value + } + queryStringParts.add("$osPath:$wrappedValue") + } + } else { + val escaped = escapeQueryStringValue(term.value) + queryStringParts.add(if (term.quoted) "\"$escaped\"" else "*$escaped*") + + val pattern = "*${term.value}*" + caseFields.filterNotNull().forEach { caseField -> + unqualifiedCaseFieldQueries.add( + QueryBuilders.wildcardQuery(caseField, pattern).caseInsensitive(true) + ) + } + } + } + + if (qualifiedCaseFieldQueries.isEmpty() && unqualifiedCaseFieldQueries.isEmpty() && queryStringParts.isEmpty()) { + return QueryBuilders.matchAllQuery() + } + + val boolQuery = QueryBuilders.boolQuery() + + qualifiedCaseFieldQueries.forEach { boolQuery.must(it) } + + if (queryStringParts.isNotEmpty()) { + val docQuery = QueryBuilders.queryStringQuery(queryStringParts.joinToString(" AND ")) + .apply { docFields.forEach { field(it) } } + .lenient(true) + .analyzeWildcard(true) + .defaultOperator(Operator.AND) + + if (unqualifiedCaseFieldQueries.isNotEmpty()) { + val shouldQuery = QueryBuilders.boolQuery() + .should(docQuery) + unqualifiedCaseFieldQueries.forEach { shouldQuery.should(it) } + shouldQuery.minimumShouldMatch(1) + boolQuery.must(shouldQuery) + } else { + boolQuery.must(docQuery) + } + } else if (unqualifiedCaseFieldQueries.isNotEmpty()) { + unqualifiedCaseFieldQueries.forEach { boolQuery.must(it) } + } + + return boolQuery + } + + private fun buildGlobalSearchQueryForAllDefinitions(query: String): QueryBuilder { + val accessibleDefinitions = caseDefinitionService.getCaseDefinitions(active = true) + if (accessibleDefinitions.isEmpty()) { + return MATCH_NONE + } + + val definitionQueries = accessibleDefinitions.mapNotNull { definition -> + val searchFields = runWithoutAuthorization { + searchFieldService.getSearchFields(definition.id.key) + } + if (searchFields.isEmpty()) { + null + } else { + QueryBuilders.boolQuery() + .must(QueryBuilders.termQuery(DEFINITION_NAME_FIELD, definition.id.key)) + .must(buildGlobalSearchQuery(query, searchFields)) + } + } + + if (definitionQueries.isEmpty()) { + return MATCH_NONE + } + + return QueryBuilders.boolQuery().apply { + definitionQueries.forEach { should(it) } + minimumShouldMatch(1) + } + } + + private data class ParsedTerm( + val field: String?, + val value: String, + val quoted: Boolean + ) + + private fun parseGlobalSearch(query: String): List { + val terms = mutableListOf() + val fieldPattern = """(\w+(?:\.\w+)*):("([^"]+)"|(\S+))""".toRegex() + + var remaining = query + var lastEnd = 0 + + for (match in fieldPattern.findAll(query)) { + val before = query.substring(lastEnd, match.range.first).trim() + if (before.isNotEmpty()) { + terms.addAll(parseUnqualifiedTerms(before)) + } + + val fieldName = match.groupValues[1] + val quoted = match.groupValues[3].isNotEmpty() + val value = if (quoted) match.groupValues[3] else match.groupValues[4] + + terms.add(ParsedTerm(fieldName, value, quoted)) + lastEnd = match.range.last + 1 + } + + val after = query.substring(lastEnd).trim() + if (after.isNotEmpty()) { + terms.addAll(parseUnqualifiedTerms(after)) + } + + return terms + } + + private fun parseUnqualifiedTerms(text: String): List { + val terms = mutableListOf() + val quotedPattern = """"([^"]+)"""".toRegex() + + var remaining = text + var lastEnd = 0 + + for (match in quotedPattern.findAll(text)) { + val before = text.substring(lastEnd, match.range.first).trim() + if (before.isNotEmpty()) { + before.split("\\s+".toRegex()).filter { it.isNotEmpty() }.forEach { + terms.add(ParsedTerm(null, it, false)) + } + } + terms.add(ParsedTerm(null, match.groupValues[1], true)) + lastEnd = match.range.last + 1 + } + + val after = text.substring(lastEnd).trim() + if (after.isNotEmpty()) { + after.split("\\s+".toRegex()).filter { it.isNotEmpty() }.forEach { + terms.add(ParsedTerm(null, it, false)) + } + } + + return terms + } + + private fun escapeQueryStringValue(value: String): String { + val specialChars = """[\+\-\=\&\|\!\(\)\{\}\[\]\^\~\*\?\:\\\/]""".toRegex() + return value.replace(specialChars) { "\\${it.value}" } + } + + private fun removePrefixes(path: String?): String? { + return path?.removePrefix(DOC_PREFIX)?.removePrefix(CASE_PREFIX) + } + + companion object { + private const val DOC_PREFIX = "doc:" + private const val CASE_PREFIX = "case:" + private const val DEFINITION_NAME_FIELD = "definitionId.name" + private const val BLUEPRINT_TYPE_FIELD = "definitionId.blueprintId.blueprintType" + private val MATCH_NONE: QueryBuilder = QueryBuilders.boolQuery().mustNot(QueryBuilders.matchAllQuery()) + + private fun AdvancedSearchRequest.OtherFilter.rangeFromValue(): Any? = + AdvancedSearchRequest.OtherFilter::class.java.getMethod("getRangeFrom").invoke(this) + + private fun AdvancedSearchRequest.OtherFilter.rangeToValue(): Any? = + AdvancedSearchRequest.OtherFilter::class.java.getMethod("getRangeTo").invoke(this) + } +} diff --git a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/JsonSchemaDocumentOsConverter.kt b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/JsonSchemaDocumentOsConverter.kt new file mode 100644 index 0000000000..1dc9a9a164 --- /dev/null +++ b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/JsonSchemaDocumentOsConverter.kt @@ -0,0 +1,104 @@ +/* + * Copyright 2015-2026 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.ritense.document.opensearch.service + +import com.fasterxml.jackson.databind.JsonNode +import com.fasterxml.jackson.databind.ObjectMapper +import com.ritense.document.domain.impl.JsonSchemaDocument +import com.ritense.document.opensearch.domain.JsonSchemaDocumentOsDocument +import com.ritense.document.opensearch.repository.JsonSchemaDocumentOpenSearchRepository +import io.github.oshai.kotlinlogging.KotlinLogging +import org.springframework.data.elasticsearch.BulkFailureException +import org.springframework.data.elasticsearch.VersionConflictException + +/** + * Single, shared implementation of the [JsonSchemaDocument] → [JsonSchemaDocumentOsDocument] conversion + * and of the poison-pill-isolated bulk indexing. Reused by the live event listener, the reconciler and + * the admin re-index service so all three writers produce byte-identical OpenSearch documents and share + * the same failure isolation. + */ +open class JsonSchemaDocumentOsConverter( + private val objectMapper: ObjectMapper, + private val openSearchRepository: JsonSchemaDocumentOpenSearchRepository, +) { + + /** + * Converts a JPA [JsonSchemaDocument] to its OpenSearch read model. The lazy `internalStatus` + * association (and the eager `caseTags`) must be initialised before calling this — either by an + * ambient transaction or by an eager fetch — otherwise serialization sees a null status. + */ + open fun toOsDocument(document: JsonSchemaDocument): JsonSchemaDocumentOsDocument { + val tree = objectMapper.valueToTree(document) + return objectMapper.treeToValue(tree, JsonSchemaDocumentOsDocument::class.java) + .copy( + // Extract content text directly from the document's content, not from the serialized tree + // (the tree may not include content if the getter isn't JavaBean-named). + contentText = extractLeafValues(document.content().asJson()), + // JPA optimistic-lock counter drives the OpenSearch external version; +1 keeps it ≥ 1. + indexVersion = (document.version() ?: 0).toLong() + 1, + ) + } + + /** + * Indexes a single bulk chunk. On an item-level [BulkFailureException] only the documents that actually + * failed are re-processed one-by-one (never the whole chunk). A version conflict (HTTP 409) is benign — + * external versioning means the stored document is already at an equal-or-newer version, so it is + * silently ignored rather than warned/counted (in the steady state the reconciler's re-sends are all + * conflicts). Any other per-document failure is isolated and counted as a skip so one poison document + * can never loop a run forever. Transport/connection errors are NOT caught here: they propagate so the + * caller can react (mark the run FAILED, park the watermark, …). + * + * @return the number of documents skipped in this chunk + */ + open fun indexChunk(chunk: List): Long = + try { + openSearchRepository.saveAll(chunk) + 0L + } catch (e: BulkFailureException) { + val byId = chunk.associateBy { it.id } + var skipped = 0L + e.failedDocuments.forEach { (id, details) -> + if (isVersionConflict(details)) return@forEach // benign: stored doc already ≥ this version + val document = byId[id] ?: return@forEach + try { + openSearchRepository.save(document) + } catch (ex: VersionConflictException) { + // benign: a newer/equal version won the race and is already indexed + } catch (ex: Exception) { + skipped++ + logger.warn(ex) { "Failed to index document $id — skipping" } + } + } + skipped + } + + companion object { + private val logger = KotlinLogging.logger {} + + /** OpenSearch bulk payload size, decoupled from any DB page size. */ + const val BULK_CHUNK_SIZE = 500 + + /** + * A bulk item failure that is an OpenSearch external-version conflict: HTTP 409, or the engine's + * `version_conflict_engine_exception` in the message as a fallback. These are expected under + * external versioning and must not be warned or counted as skips. + */ + private fun isVersionConflict(details: BulkFailureException.FailureDetails): Boolean = + details.status() == 409 || + details.errorMessage()?.contains("version_conflict_engine_exception") == true + } +} diff --git a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/OpenSearchHealthService.kt b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/OpenSearchHealthService.kt new file mode 100644 index 0000000000..024d4f9a01 --- /dev/null +++ b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/OpenSearchHealthService.kt @@ -0,0 +1,58 @@ +/* + * Copyright 2015-2024 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.ritense.document.opensearch.service + +import com.ritense.document.opensearch.OpenSearchProperties +import io.github.oshai.kotlinlogging.KotlinLogging +import org.opensearch.client.RequestOptions +import org.opensearch.client.RestHighLevelClient + +class OpenSearchHealthService( + private val restHighLevelClient: RestHighLevelClient, + private val toggle: SearchEngineToggle, + private val properties: OpenSearchProperties, +) { + + fun checkAndRecover() { + if (!toggle.isFallbackActive()) { + return + } + + val available = try { + restHighLevelClient.ping(RequestOptions.DEFAULT) + } catch (_: Exception) { + false + } + + if (available) { + logger.info { "OpenSearch is available again, deactivating fallback" } + toggle.deactivateFallback() + } else { + logFallbackWarningIfNeeded() + } + } + + private fun logFallbackWarningIfNeeded() { + if (toggle.shouldLogWarning(properties.fallbackWarningIntervalMs)) { + logger.warn { "OpenSearch unavailable, using PostgreSQL fallback" } + } + } + + companion object { + private val logger = KotlinLogging.logger {} + } +} diff --git a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/OpenSearchReindexRunService.kt b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/OpenSearchReindexRunService.kt new file mode 100644 index 0000000000..155faa4687 --- /dev/null +++ b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/OpenSearchReindexRunService.kt @@ -0,0 +1,280 @@ +/* + * Copyright 2015-2026 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.ritense.document.opensearch.service + +import com.fasterxml.jackson.databind.ObjectMapper +import com.ritense.document.opensearch.OpenSearchProperties +import com.ritense.document.domain.impl.JsonSchemaDocument +import com.ritense.document.opensearch.domain.OpenSearchReindexRun +import com.ritense.document.opensearch.domain.ReindexRunStatus +import com.ritense.document.opensearch.repository.OpenSearchReindexRunRepository +import io.github.oshai.kotlinlogging.KotlinLogging +import jakarta.persistence.EntityManager +import jakarta.persistence.criteria.Predicate +import org.springframework.boot.context.event.ApplicationReadyEvent +import org.springframework.context.event.EventListener +import org.springframework.data.domain.Page +import org.springframework.data.domain.PageImpl +import org.springframework.data.domain.Pageable +import org.springframework.transaction.annotation.Transactional +import java.time.Duration +import java.time.Instant +import java.time.LocalDateTime +import java.util.UUID +import java.util.concurrent.ConcurrentHashMap + +/** + * Thin transactional wrapper around [OpenSearchReindexRunRepository] for managing re-index run state. + * + * Each mutation runs in its own (short-lived) transaction, independent of the read-only document-fetch + * transactions in [DocumentOpenSearchReindexService], so progress and status are committed and visible + * across instances as the run proceeds. + */ +@Transactional +open class OpenSearchReindexRunService( + private val repository: OpenSearchReindexRunRepository, + private val objectMapper: ObjectMapper, + private val properties: OpenSearchProperties, + private val entityManager: EntityManager, +) { + + private val totalCountCache = ConcurrentHashMap>() + + /** + * On startup, reconcile any [ReindexRunStatus.RUNNING] row whose heartbeat has gone stale (older than + * [OpenSearchProperties.Reindex.runningHeartbeatTimeout]): the instance that owned it has crashed or + * restarted and can no longer be advancing it. Mark them FAILED (resumable from their cursor). Runs still + * being advanced by a live instance keep a fresh heartbeat and are left untouched, so this is cluster-safe. + */ + @EventListener(ApplicationReadyEvent::class) + open fun reconcileOrphanedRuns() { + val staleBefore = LocalDateTime.now().minus(properties.reindex.runningHeartbeatTimeout) + val orphaned = repository.findAllByStatusAndHeartbeatOnBefore(ReindexRunStatus.RUNNING, staleBefore) + if (orphaned.isEmpty()) return + val now = LocalDateTime.now() + orphaned.forEach { it.fail(now, "Reconciled on startup: RUNNING run with a stale heartbeat") } + repository.saveAll(orphaned) + logger.warn { "Reconciled ${orphaned.size} orphaned RUNNING re-index run(s) with a stale heartbeat to FAILED" } + } + + /** + * Creates a fresh RUNNING run for [request], or — when [ReindexRequest.resumeRunId] is set — + * re-arms that existing run so it continues from its persisted [OpenSearchReindexRun.lastId]. + */ + open fun startOrResume(request: ReindexRequest): OpenSearchReindexRun { + request.resumeRunId?.let { resumeRunId -> + val existing = repository.findById(resumeRunId).orElseThrow { + IllegalArgumentException("No re-index run found for resumeRunId=$resumeRunId") + } + existing.resume(LocalDateTime.now()) + return repository.save(existing) + } + return repository.save( + OpenSearchReindexRun( + id = UUID.randomUUID(), + status = ReindexRunStatus.RUNNING, + scope = serializeScope(request), + pageSize = request.effectivePageSize(), + startedOn = LocalDateTime.now(), + heartbeatOn = LocalDateTime.now(), + ) + ) + } + + /** + * Whether an admin (re)index run is currently in progress anywhere in the cluster. A [ReindexRunStatus.RUNNING] + * row only counts while its heartbeat is fresher than [heartbeatTimeout], so a run left behind by a crashed + * instance cannot report "running" forever. Used to temporarily route document search to PostgreSQL while the + * index is being filled. + */ + @Transactional(readOnly = true) + open fun isReindexRunning(heartbeatTimeout: Duration): Boolean = + repository.existsByStatusAndHeartbeatOnAfter( + ReindexRunStatus.RUNNING, + LocalDateTime.now().minus(heartbeatTimeout), + ) + + @Transactional(readOnly = true) + open fun cursorOf(runId: UUID): UUID? = requireRun(runId).lastId + + @Transactional(readOnly = true) + open fun processedOf(runId: UUID): Long = requireRun(runId).processedCount + + open fun recordProgress(runId: UUID, lastId: UUID?, processed: Long, skipped: Long) { + val run = requireRun(runId) + run.recordProgress(lastId, processed, skipped, LocalDateTime.now()) + repository.save(run) + } + + open fun complete(runId: UUID, totalCount: Long?) { + val run = requireRun(runId) + run.totalCount = totalCount + run.complete(LocalDateTime.now()) + repository.save(run) + totalCountCache.remove(runId) + } + + open fun fail(runId: UUID, error: String?) { + val run = requireRun(runId) + run.fail(LocalDateTime.now(), error) + repository.save(run) + totalCountCache.remove(runId) + } + + open fun stop(runId: UUID) { + val run = requireRun(runId) + run.stop(LocalDateTime.now()) + repository.save(run) + totalCountCache.remove(runId) + } + + open fun recordPruned(runId: UUID, pruned: Long) { + val run = requireRun(runId) + run.prunedCount = pruned + repository.save(run) + } + + open fun startPruning(runId: UUID, totalOsCount: Long) { + val run = requireRun(runId) + run.startPruning(totalOsCount, LocalDateTime.now()) + repository.save(run) + } + + open fun recordPruneProgress(runId: UUID, checked: Long, pruned: Long) { + val run = requireRun(runId) + run.recordPruneProgress(checked, pruned, LocalDateTime.now()) + repository.save(run) + } + + /** + * Status of a specific run (by [runId]) or — when null — of the most recent run. Returns a + * not-running placeholder when no matching run exists. + */ + @Transactional(readOnly = true) + open fun toStatusMap(runId: UUID?): Map { + val run = (if (runId != null) repository.findById(runId).orElse(null) + else repository.findFirstByOrderByStartedOnDesc()) + ?: return mapOf("running" to false, "runId" to null) + return toMap(run) + } + + @Transactional(readOnly = true) + open fun listRuns(pageable: Pageable): Page> { + val page = repository.findAllByOrderByStartedOnDesc(pageable) + return PageImpl(page.content.map { toMap(it) }, pageable, page.totalElements) + } + + @Transactional(readOnly = true) + open fun scopeOf(runId: UUID): ReindexRequest = + deserializeScopeToRequest(requireRun(runId).scope) + ?: ReindexRequest() + + @Transactional(readOnly = true) + open fun pageSizeOf(runId: UUID): Int = requireRun(runId).pageSize + + private fun requireRun(runId: UUID): OpenSearchReindexRun = + repository.findById(runId).orElseThrow { IllegalArgumentException("No re-index run found for runId=$runId") } + + private fun toMap(run: OpenSearchReindexRun): Map { + val elapsedSeconds = Duration.between(run.startedOn, run.finishedOn ?: LocalDateTime.now()).seconds + val scope = deserializeScopeToRequest(run.scope) + return mapOf( + "runId" to run.id, + "status" to run.status, + "running" to (run.status == ReindexRunStatus.RUNNING), + "scope" to scope?.let { objectMapper.convertValue(it, Map::class.java) }, + "pageSize" to run.pageSize, + "lastId" to run.lastId, + "processedCount" to run.processedCount, + "skippedCount" to run.skippedCount, + "prunedCount" to run.prunedCount, + "pruneCheckedCount" to run.pruneCheckedCount, + "pruneTotalCount" to run.pruneTotalCount, + "pruningPhase" to run.pruningPhase, + "totalCount" to getTotalCount(run, scope), + "startedOn" to run.startedOn, + "heartbeatOn" to run.heartbeatOn, + "finishedOn" to run.finishedOn, + "elapsedSeconds" to elapsedSeconds, + "error" to run.error, + ) + } + + private fun getTotalCount(run: OpenSearchReindexRun, scope: ReindexRequest?): Long { + run.totalCount?.let { return it } + + if (run.status == ReindexRunStatus.RUNNING) { + val cached = totalCountCache[run.id] + if (cached != null && Instant.now().isBefore(cached.second)) { + return cached.first + } + } + + val count = countDocuments(scope) + + if (run.status == ReindexRunStatus.RUNNING) { + totalCountCache[run.id] = count to Instant.now().plus(TOTAL_COUNT_CACHE_TTL) + } + + return count + } + + private fun serializeScope(request: ReindexRequest): String? = + try { + objectMapper.writeValueAsString(request) + } catch (e: Exception) { + logger.warn(e) { "Failed to serialize re-index scope — storing null" } + null + } + + private fun deserializeScopeToRequest(scope: String?): ReindexRequest? = + scope?.let { + try { + objectMapper.readValue(it, ReindexRequest::class.java) + } catch (e: Exception) { + logger.warn(e) { "Failed to deserialize scope to ReindexRequest" } + null + } + } + + private fun countDocuments(scope: ReindexRequest?): Long { + val cb = entityManager.criteriaBuilder + val query = cb.createQuery(Long::class.java) + val root = query.from(JsonSchemaDocument::class.java) + query.select(cb.count(root)) + + val predicates = mutableListOf() + scope?.modifiedAfter?.let { predicates += cb.greaterThan(root.get("modifiedOn"), it) } + scope?.modifiedBefore?.let { predicates += cb.lessThan(root.get("modifiedOn"), it) } + scope?.documentDefinitionName?.let { + predicates += cb.equal(root.get("documentDefinitionId").get("name"), it) + } + scope?.documentIds?.takeIf { it.isNotEmpty() }?.let { + predicates += root.get("id").get("id").`in`(it) + } + + if (predicates.isNotEmpty()) { + query.where(*predicates.toTypedArray()) + } + return entityManager.createQuery(query).singleResult + } + + companion object { + private val logger = KotlinLogging.logger {} + private val TOTAL_COUNT_CACHE_TTL: Duration = Duration.ofMinutes(5) + } +} diff --git a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/ReindexProgressGate.kt b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/ReindexProgressGate.kt new file mode 100644 index 0000000000..3c4367ef91 --- /dev/null +++ b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/ReindexProgressGate.kt @@ -0,0 +1,60 @@ +/* + * Copyright 2015-2026 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.ritense.document.opensearch.service + +import com.ritense.document.opensearch.OpenSearchProperties + +/** + * Tells the [DelegatingDocumentSearchService] whether an admin (re)index run is currently filling the + * index, in which case search should temporarily fall back to PostgreSQL so users never query a + * partially-filled index. The decision is derived from the cluster-shared reindex-run state, so every node + * falls back regardless of which one runs the job, and search returns to OpenSearch automatically once all + * runs finish. + * + * The result is cached for [CACHE_TTL_MS] to avoid a database round-trip on every search — a sub-second + * delay before switching back to OpenSearch is harmless. + */ +open class ReindexProgressGate( + private val reindexRunService: OpenSearchReindexRunService, + private val properties: OpenSearchProperties, + private val clock: () -> Long = System::currentTimeMillis, +) { + + @Volatile + private var cachedResult = false + + @Volatile + private var cachedAtMillis = 0L + + @Volatile + private var initialized = false + + open fun isReindexInProgress(): Boolean { + if (!properties.reindex.fallbackToPostgresWhileRunning) return false + val now = clock() + if (!initialized || now - cachedAtMillis >= CACHE_TTL_MS) { + cachedResult = reindexRunService.isReindexRunning(properties.reindex.runningHeartbeatTimeout) + cachedAtMillis = now + initialized = true + } + return cachedResult + } + + companion object { + const val CACHE_TTL_MS = 1000L + } +} diff --git a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/ReindexRequest.kt b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/ReindexRequest.kt new file mode 100644 index 0000000000..bdab53495a --- /dev/null +++ b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/ReindexRequest.kt @@ -0,0 +1,48 @@ +/* + * Copyright 2015-2026 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.ritense.document.opensearch.service + +import java.time.LocalDateTime +import java.util.UUID + +/** + * Describes the scope of a re-index run. All filters are optional; an empty request re-indexes every + * [com.ritense.document.domain.impl.JsonSchemaDocument] into the live `json_schema_document` index. + * + * @param modifiedAfter only documents with `modifiedOn` strictly after this instant + * @param modifiedBefore only documents with `modifiedOn` strictly before this instant + * @param documentDefinitionName only documents of this document-definition name + * @param documentIds explicit subset of document ids + * @param pageSize DB keyset page size, clamped to [1, MAX_PAGE_SIZE] by [effectivePageSize] + * @param resumeRunId continue a prior (FAILED/STOPPED) run from its persisted cursor instead of starting fresh + */ +data class ReindexRequest( + val modifiedAfter: LocalDateTime? = null, + val modifiedBefore: LocalDateTime? = null, + val documentDefinitionName: String? = null, + val documentIds: List? = null, + val pageSize: Int = DEFAULT_PAGE_SIZE, + val resumeRunId: UUID? = null, + val pruneOrphans: Boolean = false, +) { + fun effectivePageSize() = pageSize.coerceIn(1, MAX_PAGE_SIZE) + + companion object { + const val DEFAULT_PAGE_SIZE = 5000 + const val MAX_PAGE_SIZE = 10_000 + } +} diff --git a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/SearchEngineToggle.kt b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/SearchEngineToggle.kt new file mode 100644 index 0000000000..3a6035c1fc --- /dev/null +++ b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/service/SearchEngineToggle.kt @@ -0,0 +1,75 @@ +/* + * Copyright 2015-2024 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.ritense.document.opensearch.service + +import java.util.concurrent.atomic.AtomicBoolean +import java.util.concurrent.atomic.AtomicLong +import java.util.concurrent.atomic.AtomicReference + +class SearchEngineToggle(default: Engine = Engine.OPENSEARCH) { + + enum class Engine { OPENSEARCH, POSTGRES } + + private val active = AtomicReference(default) + private val fallbackActive = AtomicBoolean(false) + private val lastWarningTime = AtomicLong(0) + + fun get(): Engine = active.get() + + fun set(engine: Engine) { + active.set(engine) + } + + /** + * Master switch for every active OpenSearch call (reads, live-sync writes, reconcile, index creation). + * Read live on each call site so flipping the engine at runtime — via + * [com.ritense.document.opensearch.web.SearchEngineResource] — immediately (re)enables or disables all + * OpenSearch traffic without a restart. Startup forces this to [Engine.POSTGRES] when OpenSearch is + * disabled by configuration, so this single check also honours `valtimo.opensearch.enabled`. + */ + fun isOpenSearchActive(): Boolean = active.get() == Engine.OPENSEARCH + + fun isFallbackActive(): Boolean = fallbackActive.get() + + fun activateFallback() { + fallbackActive.set(true) + } + + fun deactivateFallback() { + fallbackActive.set(false) + lastWarningTime.set(0) + } + + /** + * Route document search to PostgreSQL when the engine is not OpenSearch, while an admin reindex is + * filling the index ([reindexInProgress]), or while a connection fallback is active because OpenSearch + * is unreachable. Otherwise OpenSearch serves the query. [reindexInProgress] is a supplier so the + * engine check short-circuits it — the (potentially DB-backed) reindex check is skipped entirely when + * the engine is already PostgreSQL. + */ + fun shouldUsePostgres(reindexInProgress: () -> Boolean): Boolean = + get() != Engine.OPENSEARCH || reindexInProgress() || fallbackActive.get() + + fun shouldLogWarning(intervalMs: Long): Boolean { + val now = System.currentTimeMillis() + val last = lastWarningTime.get() + if (now - last >= intervalMs) { + return lastWarningTime.compareAndSet(last, now) + } + return false + } +} diff --git a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/web/DocumentOpenSearchReindexResource.kt b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/web/DocumentOpenSearchReindexResource.kt new file mode 100644 index 0000000000..693d771f07 --- /dev/null +++ b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/web/DocumentOpenSearchReindexResource.kt @@ -0,0 +1,59 @@ +/* + * Copyright 2015-2026 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.ritense.document.opensearch.web + +import com.ritense.document.opensearch.service.DocumentOpenSearchReindexService +import com.ritense.document.opensearch.service.ReindexRequest +import org.springframework.data.domain.Page +import org.springframework.data.domain.PageRequest +import org.springframework.http.ResponseEntity +import org.springframework.web.bind.annotation.GetMapping +import org.springframework.web.bind.annotation.PathVariable +import org.springframework.web.bind.annotation.PostMapping +import org.springframework.web.bind.annotation.RequestBody +import org.springframework.web.bind.annotation.RequestMapping +import org.springframework.web.bind.annotation.RequestParam +import org.springframework.web.bind.annotation.RestController +import java.util.UUID + +@RestController +@RequestMapping("/api/management/v1/document-opensearch") +class DocumentOpenSearchReindexResource( + private val reindexService: DocumentOpenSearchReindexService, +) { + + @PostMapping("/reindex") + fun reindex(@RequestBody(required = false) request: ReindexRequest?): ResponseEntity> { + val runId = reindexService.start(request ?: ReindexRequest()) + ?: return ResponseEntity.status(409).body(mapOf("error" to "Re-index already in progress")) + return ResponseEntity.accepted().body(mapOf("status" to "started", "runId" to runId)) + } + + @GetMapping("/reindex/runs") + fun listRuns( + @RequestParam(defaultValue = "0") page: Int, + @RequestParam(defaultValue = "20") size: Int + ): ResponseEntity>> = + ResponseEntity.ok(reindexService.listRuns(PageRequest.of(page, size))) + + @GetMapping("/reindex/status") + fun status(): ResponseEntity> = ResponseEntity.ok(reindexService.status()) + + @GetMapping("/reindex/{runId}") + fun statusById(@PathVariable runId: UUID): ResponseEntity> = + ResponseEntity.ok(reindexService.status(runId)) +} diff --git a/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/web/SearchEngineResource.kt b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/web/SearchEngineResource.kt new file mode 100644 index 0000000000..afead41d82 --- /dev/null +++ b/backend/case-opensearch/src/main/kotlin/com/ritense/document/opensearch/web/SearchEngineResource.kt @@ -0,0 +1,94 @@ +/* + * Copyright 2015-2026 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.ritense.document.opensearch.web + +import com.ritense.adminsettings.service.FeatureToggleOverridesService +import com.ritense.document.opensearch.OpenSearchProperties +import com.ritense.document.opensearch.autoconfigure.DocumentOpenSearchAutoConfiguration.Companion.SEARCH_ENGINE_TOGGLE_KEY +import com.ritense.document.opensearch.service.DocumentOpenSearchIndexInitializer +import com.ritense.document.opensearch.service.SearchEngineToggle +import io.github.oshai.kotlinlogging.KotlinLogging +import org.springframework.http.HttpStatus +import org.springframework.http.ResponseEntity +import org.springframework.web.bind.annotation.GetMapping +import org.springframework.web.bind.annotation.PutMapping +import org.springframework.web.bind.annotation.RequestBody +import org.springframework.web.bind.annotation.RequestMapping +import org.springframework.web.bind.annotation.RestController + +@RestController +@RequestMapping("/api/management/v1/search-engine") +class SearchEngineResource( + private val toggle: SearchEngineToggle, + private val openSearchProperties: OpenSearchProperties, + private val featureToggleOverridesService: FeatureToggleOverridesService, + private val indexInitializer: DocumentOpenSearchIndexInitializer, +) { + + @GetMapping + fun getActive(): ResponseEntity = + ResponseEntity.ok( + SearchEngineDto( + available = openSearchProperties.enabled, + active = toggle.get().name + ) + ) + + @PutMapping + fun setActive(@RequestBody body: UpdateSearchEngineDto): ResponseEntity { + if (!openSearchProperties.enabled) { + return ResponseEntity.badRequest().build() + } + + val useOpenSearch = body.active.uppercase() == "OPENSEARCH" + + if (useOpenSearch) { + try { + indexInitializer.ensureIndex() + } catch (e: Exception) { + logger.warn(e) { "Failed to initialize OpenSearch index — is OpenSearch running?" } + return ResponseEntity.status(HttpStatus.SERVICE_UNAVAILABLE) + .body(SearchEngineDto(available = true, active = toggle.get().name)) + } + } + + featureToggleOverridesService.updateToggle(SEARCH_ENGINE_TOGGLE_KEY, useOpenSearch) + + val engine = if (useOpenSearch) SearchEngineToggle.Engine.OPENSEARCH else SearchEngineToggle.Engine.POSTGRES + toggle.set(engine) + + return ResponseEntity.ok( + SearchEngineDto( + available = true, + active = toggle.get().name + ) + ) + } + + data class SearchEngineDto( + val available: Boolean, + val active: String + ) + + data class UpdateSearchEngineDto( + val active: String + ) + + companion object { + private val logger = KotlinLogging.logger {} + } +} diff --git a/backend/case-opensearch/src/main/resources/META-INF/spring.factories b/backend/case-opensearch/src/main/resources/META-INF/spring.factories new file mode 100644 index 0000000000..622a4db2ce --- /dev/null +++ b/backend/case-opensearch/src/main/resources/META-INF/spring.factories @@ -0,0 +1,2 @@ +org.springframework.boot.autoconfigure.AutoConfigurationImportFilter=\ + com.ritense.document.opensearch.autoconfigure.ExcludeElasticsearchAutoConfigurationFilter \ No newline at end of file diff --git a/backend/case-opensearch/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports b/backend/case-opensearch/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports new file mode 100644 index 0000000000..929aee7b3b --- /dev/null +++ b/backend/case-opensearch/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports @@ -0,0 +1 @@ +com.ritense.document.opensearch.autoconfigure.DocumentOpenSearchAutoConfiguration diff --git a/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/BaseOpenSearchIntegrationTest.kt b/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/BaseOpenSearchIntegrationTest.kt new file mode 100644 index 0000000000..015857aaee --- /dev/null +++ b/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/BaseOpenSearchIntegrationTest.kt @@ -0,0 +1,186 @@ +/* + * Copyright 2015-2026 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.ritense.document.opensearch + +import com.fasterxml.jackson.databind.ObjectMapper +import com.ritense.audit.service.AuditEventProcessor +import com.ritense.authorization.permission.ConditionContainer +import com.ritense.authorization.permission.Permission +import com.ritense.authorization.permission.PermissionRepository +import com.ritense.authorization.role.Role +import com.ritense.authorization.role.RoleRepository +import com.ritense.document.domain.impl.JsonSchemaDocumentDefinition +import com.ritense.document.domain.impl.snapshot.JsonSchemaDocumentSnapshot +import com.ritense.document.domain.impl.searchfield.SearchField +import com.ritense.document.opensearch.domain.JsonSchemaDocumentOsDocument +import com.ritense.document.opensearch.repository.JsonSchemaDocumentOpenSearchRepository +import com.ritense.document.opensearch.service.SearchEngineToggle +import com.ritense.document.service.impl.JsonSchemaDocumentService +import com.ritense.document.service.JsonSchemaDocumentActionProvider +import com.ritense.document.service.JsonSchemaDocumentDefinitionActionProvider +import com.ritense.document.service.JsonSchemaDocumentSnapshotActionProvider +import com.ritense.document.service.SearchFieldActionProvider +import com.ritense.outbox.OutboxService +import com.ritense.testutilscommon.junit.extension.LiquibaseRunnerExtension +import com.ritense.valtimo.contract.authentication.TeamManagementService +import com.ritense.valtimo.contract.authentication.UserManagementService +import com.ritense.valtimo.contract.mail.MailSender +import com.ritense.valtimo.service.ProcessDefinitionCaseDefinitionLinker +import org.springframework.data.elasticsearch.core.ElasticsearchOperations +import org.junit.jupiter.api.AfterEach +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Tag +import org.junit.jupiter.api.extension.ExtendWith +import org.mockito.Answers +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.boot.test.context.SpringBootTest +import org.springframework.context.event.SimpleApplicationEventMulticaster +import org.springframework.test.context.bean.override.mockito.MockitoBean +import org.springframework.test.context.bean.override.mockito.MockitoSpyBean +import org.springframework.test.context.junit.jupiter.SpringExtension +import org.springframework.transaction.annotation.Transactional +import java.util.UUID + +@SpringBootTest +@ExtendWith(SpringExtension::class, LiquibaseRunnerExtension::class) +@Tag("integration") +@Transactional +abstract class BaseOpenSearchIntegrationTest { + + @MockitoBean(answers = Answers.RETURNS_DEEP_STUBS) + lateinit var userManagementService: UserManagementService + + @MockitoBean + lateinit var teamManagementService: TeamManagementService + + @MockitoBean + lateinit var applicationEventMulticaster: SimpleApplicationEventMulticaster + + @MockitoBean + lateinit var processDefinitionCaseDefinitionLinker: ProcessDefinitionCaseDefinitionLinker + + @MockitoBean + lateinit var auditEventProcessor: AuditEventProcessor + + @MockitoBean + lateinit var mailSender: MailSender + + @MockitoSpyBean + lateinit var outboxService: OutboxService + + @Autowired + lateinit var documentService: JsonSchemaDocumentService + + @Autowired + lateinit var openSearchRepository: JsonSchemaDocumentOpenSearchRepository + + @Autowired + lateinit var elasticsearchOperations: ElasticsearchOperations + + @Autowired + lateinit var roleRepository: RoleRepository + + @Autowired + lateinit var permissionRepository: PermissionRepository + + @Autowired + lateinit var objectMapper: ObjectMapper + + @Autowired + lateinit var searchEngineToggle: SearchEngineToggle + + @BeforeEach + fun setUpBase() { + setUpPermissions() + ensureIndexExists() + openSearchRepository.deleteAll() + refreshIndex() + // Disable live event listener by default to prevent async indexing from interfering with tests. + // Tests that need live sync should call searchEngineToggle.set(SearchEngineToggle.Engine.OPENSEARCH). + searchEngineToggle.set(SearchEngineToggle.Engine.POSTGRES) + } + + @AfterEach + fun tearDownBase() { + openSearchRepository.deleteAll() + refreshIndex() + // Reset toggle to default for next test class + searchEngineToggle.set(SearchEngineToggle.Engine.OPENSEARCH) + } + + private fun ensureIndexExists() { + val indexOps = elasticsearchOperations.indexOps(JsonSchemaDocumentOsDocument::class.java) + if (!indexOps.exists()) { + indexOps.create() + indexOps.putMapping(indexOps.createMapping(JsonSchemaDocumentOsDocument::class.java)) + } + } + + /** + * Forces an OpenSearch refresh so writes/deletes are immediately visible to subsequent reads. + * OpenSearch refreshes asynchronously (default 1s), which makes write-then-read assertions flaky. + */ + protected fun refreshIndex() { + elasticsearchOperations.indexOps(JsonSchemaDocumentOsDocument::class.java).refresh() + } + + private fun setUpPermissions() { + var role = roleRepository.findByKey(FULL_ACCESS_ROLE) + if (role == null) { + role = roleRepository.save(Role(UUID.randomUUID(), FULL_ACCESS_ROLE)) + } + + val permissions = listOf( + Permission(UUID.randomUUID(), com.ritense.document.domain.impl.JsonSchemaDocument::class.java, + mutableListOf(JsonSchemaDocumentActionProvider.VIEW_LIST), ConditionContainer(emptyList()), role!!), + Permission(UUID.randomUUID(), com.ritense.document.domain.impl.JsonSchemaDocument::class.java, + mutableListOf(JsonSchemaDocumentActionProvider.VIEW), ConditionContainer(emptyList()), role!!), + Permission(UUID.randomUUID(), com.ritense.document.domain.impl.JsonSchemaDocument::class.java, + mutableListOf(JsonSchemaDocumentActionProvider.MODIFY), ConditionContainer(emptyList()), role!!), + Permission(UUID.randomUUID(), com.ritense.document.domain.impl.JsonSchemaDocument::class.java, + mutableListOf(JsonSchemaDocumentActionProvider.CREATE), ConditionContainer(emptyList()), role!!), + Permission(UUID.randomUUID(), com.ritense.document.domain.impl.JsonSchemaDocument::class.java, + mutableListOf(JsonSchemaDocumentActionProvider.CLAIM), ConditionContainer(emptyList()), role!!), + Permission(UUID.randomUUID(), com.ritense.document.domain.impl.JsonSchemaDocument::class.java, + mutableListOf(JsonSchemaDocumentActionProvider.ASSIGN), ConditionContainer(emptyList()), role!!), + Permission(UUID.randomUUID(), com.ritense.document.domain.impl.JsonSchemaDocument::class.java, + mutableListOf(JsonSchemaDocumentActionProvider.ASSIGNABLE), ConditionContainer(emptyList()), role!!), + Permission(UUID.randomUUID(), com.ritense.document.domain.impl.JsonSchemaDocument::class.java, + mutableListOf(JsonSchemaDocumentActionProvider.DELETE), ConditionContainer(emptyList()), role!!), + Permission(UUID.randomUUID(), SearchField::class.java, + mutableListOf(SearchFieldActionProvider.VIEW_LIST), ConditionContainer(emptyList()), role!!), + Permission(UUID.randomUUID(), JsonSchemaDocumentDefinition::class.java, + mutableListOf(JsonSchemaDocumentDefinitionActionProvider.VIEW), ConditionContainer(emptyList()), role!!), + Permission(UUID.randomUUID(), JsonSchemaDocumentDefinition::class.java, + mutableListOf(JsonSchemaDocumentDefinitionActionProvider.VIEW_LIST), ConditionContainer(emptyList()), role!!), + Permission(UUID.randomUUID(), JsonSchemaDocumentDefinition::class.java, + mutableListOf(JsonSchemaDocumentDefinitionActionProvider.CREATE), ConditionContainer(emptyList()), role!!), + Permission(UUID.randomUUID(), JsonSchemaDocumentDefinition::class.java, + mutableListOf(JsonSchemaDocumentDefinitionActionProvider.MODIFY), ConditionContainer(emptyList()), role!!), + Permission(UUID.randomUUID(), JsonSchemaDocumentDefinition::class.java, + mutableListOf(JsonSchemaDocumentDefinitionActionProvider.DELETE), ConditionContainer(emptyList()), role!!), + Permission(UUID.randomUUID(), JsonSchemaDocumentSnapshot::class.java, + mutableListOf(JsonSchemaDocumentSnapshotActionProvider.VIEW_LIST), ConditionContainer(emptyList()), role!!), + ) + permissionRepository.saveAll(permissions) + } + + companion object { + const val FULL_ACCESS_ROLE: String = "full access role" + const val USERNAME: String = "test@test.com" + } +} diff --git a/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/OpenSearchPropertiesTest.kt b/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/OpenSearchPropertiesTest.kt new file mode 100644 index 0000000000..18090822d9 --- /dev/null +++ b/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/OpenSearchPropertiesTest.kt @@ -0,0 +1,48 @@ +/* + * Copyright 2015-2026 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.ritense.document.opensearch + +import com.ritense.document.opensearch.service.DocumentOpenSearchReconcileJob +import org.assertj.core.api.Assertions.assertThat +import org.junit.jupiter.api.Test +import org.springframework.scheduling.annotation.Scheduled +import java.time.Duration + +class OpenSearchPropertiesTest { + + @Test + fun `reconcile interval default is PT2M`() { + assertThat(OpenSearchProperties().reconcile.interval).isEqualTo(Duration.ofMinutes(2)) + } + + /** + * The reconcile job binds `fixedDelayString` with an inline default; if that default drifts from the + * property default, environments without the property set silently run on a different cadence. This + * guards the two from diverging. + */ + @Test + fun `scheduled job fixedDelay default matches the property default`() { + val scheduled = DocumentOpenSearchReconcileJob::class.java + .getDeclaredMethod("reconcile") + .getAnnotation(Scheduled::class.java) + + // e.g. "${valtimo.opensearch.reconcile.interval:PT2M}" -> "PT2M" + val default = scheduled.fixedDelayString.substringAfterLast(':').removeSuffix("}") + + assertThat(Duration.parse(default)).isEqualTo(OpenSearchProperties().reconcile.interval) + } +} diff --git a/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/TestApplication.kt b/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/TestApplication.kt new file mode 100644 index 0000000000..8ac18acfac --- /dev/null +++ b/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/TestApplication.kt @@ -0,0 +1,22 @@ +/* + * Copyright 2015-2025 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.ritense.document.opensearch + +import org.springframework.boot.autoconfigure.SpringBootApplication + +@SpringBootApplication +class TestApplication diff --git a/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/authorization/OpenSearchPermissionConditionTranslatorTest.kt b/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/authorization/OpenSearchPermissionConditionTranslatorTest.kt new file mode 100644 index 0000000000..a08c982afa --- /dev/null +++ b/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/authorization/OpenSearchPermissionConditionTranslatorTest.kt @@ -0,0 +1,529 @@ +/* + * Copyright 2015-2026 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.ritense.document.opensearch.authorization + +import com.ritense.authorization.Action +import com.ritense.authorization.AuthorizationService +import com.ritense.authorization.permission.ConditionContainer +import com.ritense.authorization.permission.Permission +import com.ritense.authorization.permission.condition.ContainerPermissionCondition +import com.ritense.authorization.permission.condition.ExpressionPermissionCondition +import com.ritense.authorization.permission.condition.FieldPermissionCondition +import com.ritense.authorization.permission.condition.PermissionConditionOperator +import com.ritense.authorization.permission.condition.PermissionConditionOperator.EQUAL_TO +import com.ritense.authorization.permission.condition.PermissionConditionOperator.GREATER_THAN +import com.ritense.authorization.permission.condition.PermissionConditionOperator.GREATER_THAN_OR_EQUAL_TO +import com.ritense.authorization.permission.condition.PermissionConditionOperator.IN +import com.ritense.authorization.permission.condition.PermissionConditionOperator.LESS_THAN +import com.ritense.authorization.permission.condition.PermissionConditionOperator.LESS_THAN_OR_EQUAL_TO +import com.ritense.authorization.permission.condition.PermissionConditionOperator.LIST_CONTAINS +import com.ritense.authorization.permission.condition.PermissionConditionOperator.NOT_EQUAL_TO +import com.ritense.authorization.role.Role +import com.ritense.authorization.specification.AuthorizationSpecification +import com.ritense.document.domain.impl.JsonSchemaDocument +import com.ritense.document.domain.impl.JsonSchemaDocumentId +import com.ritense.document.repository.impl.JsonSchemaDocumentRepository +import org.assertj.core.api.Assertions.assertThat +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test +import org.mockito.kotlin.any +import org.mockito.kotlin.mock +import org.mockito.kotlin.never +import org.mockito.kotlin.verify +import org.mockito.kotlin.whenever +import org.junit.jupiter.api.assertThrows +import org.opensearch.index.query.BoolQueryBuilder +import org.opensearch.index.query.ExistsQueryBuilder +import org.opensearch.index.query.IdsQueryBuilder +import org.opensearch.index.query.MatchAllQueryBuilder +import org.opensearch.index.query.QueryBuilders +import org.opensearch.index.query.RangeQueryBuilder +import org.opensearch.index.query.TermQueryBuilder +import org.opensearch.index.query.TermsQueryBuilder +import java.util.UUID + +class OpenSearchPermissionConditionTranslatorTest { + + private lateinit var authorizationService: AuthorizationService + private lateinit var documentRepository: JsonSchemaDocumentRepository + private lateinit var translator: OpenSearchPermissionConditionTranslator + + @BeforeEach + fun setUp() { + authorizationService = mock() + documentRepository = mock() + translator = OpenSearchPermissionConditionTranslator( + openSearchMappers = emptyList(), + authorizationService = authorizationService, + documentRepository = documentRepository, + ) + } + + @Test + fun `jpaFallback returns ids query with matching document IDs`() { + val docId1 = UUID.randomUUID() + val docId2 = UUID.randomUUID() + val doc1 = mockDocument(docId1) + val doc2 = mockDocument(docId2) + + val spec: AuthorizationSpecification = mock() + whenever(authorizationService.getAuthorizationSpecification(any(), any())) + .thenReturn(spec) + whenever(documentRepository.findAll(spec)).thenReturn(listOf(doc1, doc2)) + + val condition = ContainerPermissionCondition( + resourceType = UnmappedEntity::class.java, + conditions = listOf( + FieldPermissionCondition("someField", PermissionConditionOperator.EQUAL_TO, "someValue") + ) + ) + val permission = Permission( + resourceType = JsonSchemaDocument::class.java, + actions = mutableListOf(Action(Action.VIEW)), + conditionContainer = ConditionContainer(listOf(condition)), + role = Role(key = "test-role"), + ) + + val result = translator.toQuery(listOf(permission), Action(Action.VIEW)) + + assertThat(result).isInstanceOf(IdsQueryBuilder::class.java) + val idsQuery = result as IdsQueryBuilder + assertThat(idsQuery.ids()).containsExactlyInAnyOrder(docId1.toString(), docId2.toString()) + } + + @Test + fun `jpaFallback returns empty ids query when no documents match`() { + val spec: AuthorizationSpecification = mock() + whenever(authorizationService.getAuthorizationSpecification(any(), any())) + .thenReturn(spec) + whenever(documentRepository.findAll(spec)).thenReturn(emptyList()) + + val condition = ContainerPermissionCondition( + resourceType = UnmappedEntity::class.java, + conditions = emptyList() + ) + val permission = Permission( + resourceType = JsonSchemaDocument::class.java, + actions = mutableListOf(Action(Action.VIEW)), + conditionContainer = ConditionContainer(listOf(condition)), + role = Role(key = "test-role"), + ) + + val result = translator.toQuery(listOf(permission), Action(Action.VIEW)) + + assertThat(result).isInstanceOf(IdsQueryBuilder::class.java) + val idsQuery = result as IdsQueryBuilder + assertThat(idsQuery.ids()).isEmpty() + } + + @Test + fun `translateContainer uses mapper when available`() { + val mockMapper: OpenSearchAuthorizationEntityMapper = mock() + whenever(mockMapper.supports(JsonSchemaDocument::class.java, MappedEntity::class.java)).thenReturn(true) + whenever(mockMapper.mapQuery(any())).thenReturn(QueryBuilders.matchAllQuery()) + + val translatorWithMapper = OpenSearchPermissionConditionTranslator( + openSearchMappers = listOf(mockMapper), + authorizationService = authorizationService, + documentRepository = documentRepository, + ) + + val condition = ContainerPermissionCondition( + resourceType = MappedEntity::class.java, + conditions = emptyList() + ) + val permission = Permission( + resourceType = JsonSchemaDocument::class.java, + actions = mutableListOf(Action(Action.VIEW)), + conditionContainer = ConditionContainer(listOf(condition)), + role = Role(key = "test-role"), + ) + + val result = translatorWithMapper.toQuery(listOf(permission), Action(Action.VIEW)) + + verify(mockMapper).mapQuery(any()) + verify(authorizationService, never()).getAuthorizationSpecification(any(), any()) + assertThat(result).isInstanceOf(MatchAllQueryBuilder::class.java) + } + + @Test + fun `translateContainer falls back to JPA when no mapper supports the type`() { + val spec: AuthorizationSpecification = mock() + whenever(authorizationService.getAuthorizationSpecification(any(), any())) + .thenReturn(spec) + whenever(documentRepository.findAll(spec)).thenReturn(emptyList()) + + val condition = ContainerPermissionCondition( + resourceType = UnmappedEntity::class.java, + conditions = emptyList() + ) + val permission = Permission( + resourceType = JsonSchemaDocument::class.java, + actions = mutableListOf(Action(Action.VIEW)), + conditionContainer = ConditionContainer(listOf(condition)), + role = Role(key = "test-role"), + ) + + translator.toQuery(listOf(permission), Action(Action.VIEW)) + + verify(authorizationService).getAuthorizationSpecification(any(), any()) + verify(documentRepository).findAll(spec) + } + + // --- toQuery edge cases --- + + @Test + fun `toQuery returns deny-all when no permissions match action`() { + val permission = Permission( + resourceType = JsonSchemaDocument::class.java, + actions = mutableListOf(Action(Action.VIEW)), + conditionContainer = ConditionContainer(emptyList()), + role = Role(key = "test-role"), + ) + + val result = translator.toQuery(listOf(permission), Action(Action.DELETE)) + + assertThat(result).isInstanceOf(IdsQueryBuilder::class.java) + assertThat((result as IdsQueryBuilder).ids()).isEmpty() + } + + @Test + fun `toQuery returns deny-all when permissions list is empty`() { + val result = translator.toQuery(emptyList(), Action(Action.VIEW)) + + assertThat(result).isInstanceOf(IdsQueryBuilder::class.java) + assertThat((result as IdsQueryBuilder).ids()).isEmpty() + } + + @Test + fun `toQuery ORs multiple permissions together`() { + val permission1 = Permission( + resourceType = JsonSchemaDocument::class.java, + actions = mutableListOf(Action(Action.VIEW)), + conditionContainer = ConditionContainer(listOf( + FieldPermissionCondition("createdBy", EQUAL_TO, "user1") + )), + role = Role(key = "role1"), + ) + val permission2 = Permission( + resourceType = JsonSchemaDocument::class.java, + actions = mutableListOf(Action(Action.VIEW)), + conditionContainer = ConditionContainer(listOf( + FieldPermissionCondition("createdBy", EQUAL_TO, "user2") + )), + role = Role(key = "role2"), + ) + + val result = translator.toQuery(listOf(permission1, permission2), Action(Action.VIEW)) + + assertThat(result).isInstanceOf(BoolQueryBuilder::class.java) + val boolQuery = result as BoolQueryBuilder + assertThat(boolQuery.should()).hasSize(2) + assertThat(boolQuery.minimumShouldMatch()).isEqualTo("1") + } + + @Test + fun `toQuery ANDs multiple conditions within a permission`() { + val permission = Permission( + resourceType = JsonSchemaDocument::class.java, + actions = mutableListOf(Action(Action.VIEW)), + conditionContainer = ConditionContainer(listOf( + FieldPermissionCondition("createdBy", EQUAL_TO, "user1"), + FieldPermissionCondition("assigneeId", EQUAL_TO, "user2") + )), + role = Role(key = "test-role"), + ) + + val result = translator.toQuery(listOf(permission), Action(Action.VIEW)) + + assertThat(result).isInstanceOf(BoolQueryBuilder::class.java) + val boolQuery = result as BoolQueryBuilder + assertThat(boolQuery.must()).hasSize(2) + } + + @Test + fun `toQuery returns single query unwrapped when only one permission with one condition`() { + val permission = Permission( + resourceType = JsonSchemaDocument::class.java, + actions = mutableListOf(Action(Action.VIEW)), + conditionContainer = ConditionContainer(listOf( + FieldPermissionCondition("createdBy", EQUAL_TO, "user1") + )), + role = Role(key = "test-role"), + ) + + val result = translator.toQuery(listOf(permission), Action(Action.VIEW)) + + assertThat(result).isInstanceOf(TermQueryBuilder::class.java) + } + + // --- applyOperator tests --- + + @Test + fun `applyOperator EQUAL_TO with value returns term query`() { + val result = OpenSearchPermissionConditionTranslator.applyOperator("field", EQUAL_TO, "value") + + assertThat(result).isInstanceOf(TermQueryBuilder::class.java) + val termQuery = result as TermQueryBuilder + assertThat(termQuery.fieldName()).isEqualTo("field") + assertThat(termQuery.value()).isEqualTo("value") + } + + @Test + fun `applyOperator EQUAL_TO with null returns must-not-exists query`() { + val result = OpenSearchPermissionConditionTranslator.applyOperator("field", EQUAL_TO, null) + + assertThat(result).isInstanceOf(BoolQueryBuilder::class.java) + val boolQuery = result as BoolQueryBuilder + assertThat(boolQuery.mustNot()).hasSize(1) + assertThat(boolQuery.mustNot()[0]).isInstanceOf(ExistsQueryBuilder::class.java) + } + + @Test + fun `applyOperator NOT_EQUAL_TO with value returns must-not-term query`() { + val result = OpenSearchPermissionConditionTranslator.applyOperator("field", NOT_EQUAL_TO, "value") + + assertThat(result).isInstanceOf(BoolQueryBuilder::class.java) + val boolQuery = result as BoolQueryBuilder + assertThat(boolQuery.mustNot()).hasSize(1) + assertThat(boolQuery.mustNot()[0]).isInstanceOf(TermQueryBuilder::class.java) + } + + @Test + fun `applyOperator NOT_EQUAL_TO with null returns exists query`() { + val result = OpenSearchPermissionConditionTranslator.applyOperator("field", NOT_EQUAL_TO, null) + + assertThat(result).isInstanceOf(ExistsQueryBuilder::class.java) + } + + @Test + fun `applyOperator GREATER_THAN returns range query with gt`() { + val result = OpenSearchPermissionConditionTranslator.applyOperator("field", GREATER_THAN, 10) + + assertThat(result).isInstanceOf(RangeQueryBuilder::class.java) + val rangeQuery = result as RangeQueryBuilder + assertThat(rangeQuery.from()).isEqualTo(10) + assertThat(rangeQuery.includeLower()).isFalse() + } + + @Test + fun `applyOperator GREATER_THAN_OR_EQUAL_TO returns range query with gte`() { + val result = OpenSearchPermissionConditionTranslator.applyOperator("field", GREATER_THAN_OR_EQUAL_TO, 10) + + assertThat(result).isInstanceOf(RangeQueryBuilder::class.java) + val rangeQuery = result as RangeQueryBuilder + assertThat(rangeQuery.from()).isEqualTo(10) + assertThat(rangeQuery.includeLower()).isTrue() + } + + @Test + fun `applyOperator LESS_THAN returns range query with lt`() { + val result = OpenSearchPermissionConditionTranslator.applyOperator("field", LESS_THAN, 10) + + assertThat(result).isInstanceOf(RangeQueryBuilder::class.java) + val rangeQuery = result as RangeQueryBuilder + assertThat(rangeQuery.to()).isEqualTo(10) + assertThat(rangeQuery.includeUpper()).isFalse() + } + + @Test + fun `applyOperator LESS_THAN_OR_EQUAL_TO returns range query with lte`() { + val result = OpenSearchPermissionConditionTranslator.applyOperator("field", LESS_THAN_OR_EQUAL_TO, 10) + + assertThat(result).isInstanceOf(RangeQueryBuilder::class.java) + val rangeQuery = result as RangeQueryBuilder + assertThat(rangeQuery.to()).isEqualTo(10) + assertThat(rangeQuery.includeUpper()).isTrue() + } + + @Test + fun `applyOperator LIST_CONTAINS returns term query`() { + val result = OpenSearchPermissionConditionTranslator.applyOperator("field", LIST_CONTAINS, "value") + + assertThat(result).isInstanceOf(TermQueryBuilder::class.java) + } + + @Test + fun `applyOperator IN returns terms query`() { + val result = OpenSearchPermissionConditionTranslator.applyOperator("field", IN, listOf("a", "b", "c")) + + assertThat(result).isInstanceOf(TermsQueryBuilder::class.java) + } + + @Test + fun `applyOperator IN throws when value is not a collection`() { + assertThrows { + OpenSearchPermissionConditionTranslator.applyOperator("field", IN, "not-a-collection") + } + } + + // --- Field and expression translation --- + + @Test + fun `translateField maps JPA field to OpenSearch field`() { + val permission = Permission( + resourceType = JsonSchemaDocument::class.java, + actions = mutableListOf(Action(Action.VIEW)), + conditionContainer = ConditionContainer(listOf( + FieldPermissionCondition("content.content", EQUAL_TO, 123) + )), + role = Role(key = "test-role"), + ) + + val result = translator.toQuery(listOf(permission), Action(Action.VIEW)) + + assertThat(result).isInstanceOf(TermQueryBuilder::class.java) + val termQuery = result as TermQueryBuilder + assertThat(termQuery.fieldName()).isEqualTo("content") + } + + @Test + fun `translateField adds keyword suffix for string content fields`() { + val permission = Permission( + resourceType = JsonSchemaDocument::class.java, + actions = mutableListOf(Action(Action.VIEW)), + conditionContainer = ConditionContainer(listOf( + ExpressionPermissionCondition("content", "$.name", EQUAL_TO, "John", String::class.java) + )), + role = Role(key = "test-role"), + ) + + val result = translator.toQuery(listOf(permission), Action(Action.VIEW)) + + assertThat(result).isInstanceOf(TermQueryBuilder::class.java) + val termQuery = result as TermQueryBuilder + assertThat(termQuery.fieldName()).isEqualTo("content.name.keyword") + } + + @Test + fun `translateExpression does not add keyword suffix for non-string values`() { + val permission = Permission( + resourceType = JsonSchemaDocument::class.java, + actions = mutableListOf(Action(Action.VIEW)), + conditionContainer = ConditionContainer(listOf( + ExpressionPermissionCondition("content", "$.age", EQUAL_TO, 25, Int::class.java) + )), + role = Role(key = "test-role"), + ) + + val result = translator.toQuery(listOf(permission), Action(Action.VIEW)) + + assertThat(result).isInstanceOf(TermQueryBuilder::class.java) + val termQuery = result as TermQueryBuilder + assertThat(termQuery.fieldName()).isEqualTo("content.age") + } + + @Test + fun `translateExpression does not add keyword suffix for range operators`() { + val permission = Permission( + resourceType = JsonSchemaDocument::class.java, + actions = mutableListOf(Action(Action.VIEW)), + conditionContainer = ConditionContainer(listOf( + ExpressionPermissionCondition("content", "$.name", GREATER_THAN, "A", String::class.java) + )), + role = Role(key = "test-role"), + ) + + val result = translator.toQuery(listOf(permission), Action(Action.VIEW)) + + assertThat(result).isInstanceOf(RangeQueryBuilder::class.java) + val rangeQuery = result as RangeQueryBuilder + assertThat(rangeQuery.fieldName()).isEqualTo("content.name") + } + + // --- Helper methods --- + + @Test + fun `jpaToOsField returns mapped field name`() { + assertThat(OpenSearchPermissionConditionTranslator.jpaToOsField("content.content")).isEqualTo("content") + assertThat(OpenSearchPermissionConditionTranslator.jpaToOsField("createdBy")).isEqualTo("createdBy") + assertThat(OpenSearchPermissionConditionTranslator.jpaToOsField("assigneeId")).isEqualTo("assigneeId") + } + + @Test + fun `jpaToOsField returns original field name when no mapping exists`() { + assertThat(OpenSearchPermissionConditionTranslator.jpaToOsField("unmappedField")).isEqualTo("unmappedField") + } + + @Test + fun `isDynamicTextField returns true for string content field with term operator`() { + assertThat(OpenSearchPermissionConditionTranslator.isDynamicTextField("content.name", EQUAL_TO, "value")).isTrue() + assertThat(OpenSearchPermissionConditionTranslator.isDynamicTextField("content.name", NOT_EQUAL_TO, "value")).isTrue() + assertThat(OpenSearchPermissionConditionTranslator.isDynamicTextField("content.name", LIST_CONTAINS, "value")).isTrue() + assertThat(OpenSearchPermissionConditionTranslator.isDynamicTextField("content.name", IN, listOf("a", "b"))).isTrue() + } + + @Test + fun `isDynamicTextField returns false for non-content fields`() { + assertThat(OpenSearchPermissionConditionTranslator.isDynamicTextField("createdBy", EQUAL_TO, "value")).isFalse() + } + + @Test + fun `isDynamicTextField returns false for null value`() { + assertThat(OpenSearchPermissionConditionTranslator.isDynamicTextField("content.name", EQUAL_TO, null)).isFalse() + } + + @Test + fun `isDynamicTextField returns false for non-term operators`() { + assertThat(OpenSearchPermissionConditionTranslator.isDynamicTextField("content.name", GREATER_THAN, "value")).isFalse() + assertThat(OpenSearchPermissionConditionTranslator.isDynamicTextField("content.name", LESS_THAN, "value")).isFalse() + } + + @Test + fun `isDynamicTextField returns false for non-string values`() { + assertThat(OpenSearchPermissionConditionTranslator.isDynamicTextField("content.age", EQUAL_TO, 25)).isFalse() + } + + @Test + fun `translateCondition throws for unknown condition type`() { + val unknownCondition = UnknownPermissionCondition() + + val permission = Permission( + resourceType = JsonSchemaDocument::class.java, + actions = mutableListOf(Action(Action.VIEW)), + conditionContainer = ConditionContainer(listOf(unknownCondition)), + role = Role(key = "test-role"), + ) + + assertThrows { + translator.toQuery(listOf(permission), Action(Action.VIEW)) + } + } + + class UnknownPermissionCondition : com.ritense.authorization.permission.condition.PermissionCondition( + com.ritense.authorization.permission.condition.PermissionConditionType.FIELD + ) { + override fun isValid(entity: T): Boolean = true + override fun toPredicate( + root: jakarta.persistence.criteria.Root, + query: jakarta.persistence.criteria.AbstractQuery<*>, + criteriaBuilder: jakarta.persistence.criteria.CriteriaBuilder, + resourceType: Class, + queryDialectHelper: com.ritense.valtimo.contract.database.QueryDialectHelper + ): jakarta.persistence.criteria.Predicate = criteriaBuilder.conjunction() + } + + private fun mockDocument(id: UUID): JsonSchemaDocument { + val doc: JsonSchemaDocument = mock() + val docId = JsonSchemaDocumentId.existingId(id) + whenever(doc.id()).thenReturn(docId) + return doc + } + + class UnmappedEntity + class MappedEntity +} diff --git a/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/handler/DocumentOpenSearchEventListenerTest.kt b/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/handler/DocumentOpenSearchEventListenerTest.kt new file mode 100644 index 0000000000..877b7e9e82 --- /dev/null +++ b/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/handler/DocumentOpenSearchEventListenerTest.kt @@ -0,0 +1,123 @@ +/* + * Copyright 2015-2026 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.ritense.document.opensearch.handler + +import com.ritense.document.domain.impl.JsonSchemaDocumentId +import com.ritense.document.domain.impl.event.JsonSchemaDocumentCreatedEvent +import com.ritense.document.event.DocumentAssigneeChangedEvent +import com.ritense.document.event.DocumentRetentionPeriodSetEvent +import com.ritense.document.opensearch.service.DocumentOpenSearchSyncService +import com.ritense.document.opensearch.service.SearchEngineToggle +import com.ritense.valtimo.contract.event.DocumentDeletedEvent +import org.assertj.core.api.Assertions.assertThatCode +import org.junit.jupiter.api.AfterEach +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test +import org.mockito.kotlin.any +import org.mockito.kotlin.mock +import org.mockito.kotlin.never +import org.mockito.kotlin.timeout +import org.mockito.kotlin.verify +import org.mockito.kotlin.whenever +import java.util.UUID + +class DocumentOpenSearchEventListenerTest { + + private val syncService: DocumentOpenSearchSyncService = mock() + private val toggle = SearchEngineToggle(SearchEngineToggle.Engine.OPENSEARCH) + private lateinit var listener: DocumentOpenSearchEventListener + + @BeforeEach + fun setUp() { + listener = DocumentOpenSearchEventListener(syncService, toggle) + } + + @AfterEach + fun tearDown() { + listener.destroy() + } + + @Test + fun `created event enqueues an upsert of the document id`() { + val id = UUID.randomUUID() + val event: JsonSchemaDocumentCreatedEvent = mock() + whenever(event.documentId()).thenReturn(JsonSchemaDocumentId.existingId(id)) + + listener.onCreated(event) + + verify(syncService, timeout(TIMEOUT_MS)).upsertById(id) + } + + @Test + fun `assignee-changed event enqueues an upsert of the document id`() { + val id = UUID.randomUUID() + val event: DocumentAssigneeChangedEvent = mock() + whenever(event.documentId).thenReturn(id) + + listener.onAssigneeChanged(event) + + verify(syncService, timeout(TIMEOUT_MS)).upsertById(id) + } + + @Test + fun `retention-set event enqueues an upsert of the document id`() { + val id = UUID.randomUUID() + val event: DocumentRetentionPeriodSetEvent = mock() + whenever(event.getDocumentId()).thenReturn(id) + + listener.onRetentionSet(event) + + verify(syncService, timeout(TIMEOUT_MS)).upsertById(id) + } + + @Test + fun `deleted event enqueues a delete of the document id`() { + val id = UUID.randomUUID() + + listener.onDeleted(DocumentDeletedEvent(id)) + + verify(syncService, timeout(TIMEOUT_MS)).delete(id) + } + + @Test + fun `no sync happens when the engine is toggled off`() { + toggle.set(SearchEngineToggle.Engine.POSTGRES) + val id = UUID.randomUUID() + val event: JsonSchemaDocumentCreatedEvent = mock() + whenever(event.documentId()).thenReturn(JsonSchemaDocumentId.existingId(id)) + + // The engine gate is checked synchronously before the task is submitted, so no upsert is ever enqueued. + listener.onCreated(event) + + verify(syncService, never()).upsertById(any()) + } + + @Test + fun `a failing sync task never propagates to the caller`() { + val id = UUID.randomUUID() + val event: JsonSchemaDocumentCreatedEvent = mock() + whenever(event.documentId()).thenReturn(JsonSchemaDocumentId.existingId(id)) + whenever(syncService.upsertById(any())).thenThrow(RuntimeException("OpenSearch is down")) + + assertThatCode { listener.onCreated(event) }.doesNotThrowAnyException() + verify(syncService, timeout(TIMEOUT_MS)).upsertById(id) + } + + companion object { + private const val TIMEOUT_MS = 2000L + } +} diff --git a/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/service/ContentTextExtractorTest.kt b/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/service/ContentTextExtractorTest.kt new file mode 100644 index 0000000000..846ef56415 --- /dev/null +++ b/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/service/ContentTextExtractorTest.kt @@ -0,0 +1,107 @@ +/* + * Copyright 2015-2025 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.ritense.document.opensearch.service + +import com.fasterxml.jackson.databind.ObjectMapper +import org.assertj.core.api.Assertions.assertThat +import org.junit.jupiter.api.Test + +class ContentTextExtractorTest { + + private val mapper = ObjectMapper() + + @Test + fun `null input returns null`() { + assertThat(extractLeafValues(null)).isNull() + } + + @Test + fun `empty object returns null`() { + val node = mapper.readTree("{}") + assertThat(extractLeafValues(node)).isNull() + } + + @Test + fun `flat object joins all leaf values`() { + val node = mapper.readTree("""{"firstName":"John","lastName":"Doe"}""") + val result = extractLeafValues(node) + assertThat(result).contains("John") + assertThat(result).contains("Doe") + } + + @Test + fun `nested object extracts leaves recursively`() { + val node = mapper.readTree("""{"person":{"name":"Alice","city":"Utrecht"}}""") + val result = extractLeafValues(node) + assertThat(result).contains("Alice") + assertThat(result).contains("Utrecht") + } + + @Test + fun `array of primitives is extracted`() { + val node = mapper.readTree("""["apple","banana","cherry"]""") + assertThat(extractLeafValues(node)).isEqualTo("apple banana cherry") + } + + @Test + fun `array of objects extracts nested leaves`() { + val node = mapper.readTree("""[{"name":"X"},{"name":"Y"}]""") + val result = extractLeafValues(node) + assertThat(result).contains("X") + assertThat(result).contains("Y") + } + + @Test + fun `null json field values are skipped`() { + val node = mapper.readTree("""{"name":null,"city":null}""") + assertThat(extractLeafValues(node)).isNull() + } + + @Test + fun `numeric value is converted to string`() { + val node = mapper.readTree("""{"count":42}""") + assertThat(extractLeafValues(node)).isEqualTo("42") + } + + @Test + fun `boolean value is converted to string`() { + val node = mapper.readTree("""{"active":true}""") + assertThat(extractLeafValues(node)).isEqualTo("true") + } + + @Test + fun `mixed types in object are all extracted`() { + val node = mapper.readTree("""{"name":"Bob","age":30,"active":false}""") + val result = extractLeafValues(node) + assertThat(result).contains("Bob") + assertThat(result).contains("30") + assertThat(result).contains("false") + } + + @Test + fun `deeply nested structure is fully extracted`() { + val node = mapper.readTree("""{"a":{"b":{"c":"deep"}}}""") + assertThat(extractLeafValues(node)).isEqualTo("deep") + } + + @Test + fun `mixed null and non-null leaves only includes non-null values`() { + val node = mapper.readTree("""{"name":"Alice","missing":null}""") + val result = extractLeafValues(node) + assertThat(result).isEqualTo("Alice") + } +} diff --git a/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/service/DelegatingDocumentSearchServiceTest.kt b/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/service/DelegatingDocumentSearchServiceTest.kt new file mode 100644 index 0000000000..571bc962bd --- /dev/null +++ b/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/service/DelegatingDocumentSearchServiceTest.kt @@ -0,0 +1,72 @@ +/* + * Copyright 2015-2026 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.ritense.document.opensearch.service + +import com.ritense.document.domain.search.AdvancedSearchRequest +import com.ritense.document.service.DocumentSearchService +import com.ritense.valtimo.contract.blueprint.BlueprintType +import org.junit.jupiter.api.Test +import org.mockito.kotlin.any +import org.mockito.kotlin.eq +import org.mockito.kotlin.mock +import org.mockito.kotlin.never +import org.mockito.kotlin.verify +import org.mockito.kotlin.whenever + +class DelegatingDocumentSearchServiceTest { + + private val openSearchService: DocumentSearchService = mock() + private val jpaService: DocumentSearchService = mock() + private val gate: ReindexProgressGate = mock() + private val request: AdvancedSearchRequest = mock() + + @Test + fun `routes to OpenSearch when engine is OpenSearch and no reindex is in progress`() { + whenever(gate.isReindexInProgress()).thenReturn(false) + val service = delegating(SearchEngineToggle.Engine.OPENSEARCH) + + service.count("house", BlueprintType.CASE, request) + + verify(openSearchService).count(eq("house"), any(), any()) + verify(jpaService, never()).count(any(), any(), any()) + } + + @Test + fun `falls back to PostgreSQL while a reindex is in progress, even with engine OpenSearch`() { + whenever(gate.isReindexInProgress()).thenReturn(true) + val service = delegating(SearchEngineToggle.Engine.OPENSEARCH) + + service.count("house", BlueprintType.CASE, request) + + verify(jpaService).count(eq("house"), any(), any()) + verify(openSearchService, never()).count(any(), any(), any()) + } + + @Test + fun `always uses PostgreSQL when engine is Postgres, without consulting the gate`() { + val service = delegating(SearchEngineToggle.Engine.POSTGRES) + + service.count("house", BlueprintType.CASE, request) + + verify(jpaService).count(eq("house"), any(), any()) + verify(openSearchService, never()).count(any(), any(), any()) + verify(gate, never()).isReindexInProgress() + } + + private fun delegating(engine: SearchEngineToggle.Engine) = + DelegatingDocumentSearchService(openSearchService, jpaService, SearchEngineToggle(engine), gate) +} diff --git a/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchLiveSyncIntTest.kt b/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchLiveSyncIntTest.kt new file mode 100644 index 0000000000..b03febed8d --- /dev/null +++ b/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchLiveSyncIntTest.kt @@ -0,0 +1,113 @@ +/* + * Copyright 2015-2026 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.ritense.document.opensearch.service + +import com.ritense.authorization.AuthorizationContext.Companion.runWithoutAuthorization +import com.ritense.document.domain.impl.JsonSchemaDocument +import com.ritense.document.domain.impl.request.NewDocumentRequest +import com.ritense.document.opensearch.BaseOpenSearchIntegrationTest +import org.assertj.core.api.Assertions.assertThat +import org.assertj.core.api.Assertions.assertThatCode +import org.junit.jupiter.api.AfterEach +import org.junit.jupiter.api.Test +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.security.test.context.support.WithMockUser +import org.springframework.transaction.annotation.Propagation +import org.springframework.transaction.annotation.Transactional +import java.util.UUID + +/** + * Integration tests for the live-sync reload path ([DocumentOpenSearchSyncService]) against a real + * PostgreSQL + OpenSearch. Runs **non-transactionally** so documents actually commit and can be reloaded + * from the source of truth — mirroring how the [com.ritense.document.opensearch.handler.DocumentOpenSearchEventListener] + * invokes the sync service after commit. The listener→sync-service dispatch itself is unit-tested. + */ +@Transactional(propagation = Propagation.NOT_SUPPORTED) +@WithMockUser(username = BaseOpenSearchIntegrationTest.USERNAME, authorities = [BaseOpenSearchIntegrationTest.FULL_ACCESS_ROLE]) +class DocumentOpenSearchLiveSyncIntTest : BaseOpenSearchIntegrationTest() { + + @Autowired + lateinit var syncService: DocumentOpenSearchSyncService + + @AfterEach + fun cleanUp() { + runWithoutAuthorization { documentService.removeDocuments("house") } + clearIndex() + } + + @Test + fun `upsertById indexes the current committed state of the document`() { + val document = createDocument("live-street") + clearIndex() + + syncService.upsertById(document.id().id) + + refreshIndex() + val indexed = openSearchRepository.findById(document.id().toString()) + assertThat(indexed).isPresent + assertThat(indexed.get().contentText).contains("live-street") + } + + @Test + fun `upsertById reloads the lazy internalStatus of the document`() { + val document = createDocument("with-status") + runWithoutAuthorization { documentService.setInternalStatus(document.id(), "started") } + clearIndex() + + syncService.upsertById(document.id().id) + + refreshIndex() + val indexed = openSearchRepository.findById(document.id().toString()) + assertThat(indexed).isPresent + assertThat(indexed.get().internalStatus).isEqualTo("started") + } + + @Test + fun `upsertById skips a document that no longer exists`() { + val danglingId = UUID.randomUUID() + + assertThatCode { syncService.upsertById(danglingId) }.doesNotThrowAnyException() + + refreshIndex() + assertThat(openSearchRepository.findById(danglingId.toString())).isEmpty + } + + @Test + fun `delete removes the document from the index`() { + val document = createDocument("to-be-deleted") + syncService.upsertById(document.id().id) + refreshIndex() + assertThat(openSearchRepository.findById(document.id().toString())).isPresent + + syncService.delete(document.id().id) + + refreshIndex() + assertThat(openSearchRepository.findById(document.id().toString())).isEmpty + } + + private fun clearIndex() { + openSearchRepository.deleteAll() + refreshIndex() + } + + private fun createDocument(street: String): JsonSchemaDocument = + runWithoutAuthorization { + documentService.createDocument( + NewDocumentRequest("house", "house", "1.0.0", objectMapper.createObjectNode().put("street", street)) + ).resultingDocument().get() + } +} diff --git a/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchReconcileIntTest.kt b/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchReconcileIntTest.kt new file mode 100644 index 0000000000..b6e2eff9be --- /dev/null +++ b/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchReconcileIntTest.kt @@ -0,0 +1,196 @@ +/* + * Copyright 2015-2026 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.ritense.document.opensearch.service + +import com.ritense.authorization.AuthorizationContext.Companion.runWithoutAuthorization +import com.ritense.document.domain.impl.JsonSchemaDocument +import com.ritense.document.domain.impl.JsonSchemaDocumentId +import com.ritense.document.domain.impl.request.NewDocumentRequest +import com.ritense.document.opensearch.BaseOpenSearchIntegrationTest +import com.ritense.document.opensearch.domain.OpenSearchReconcileState +import com.ritense.document.opensearch.domain.PendingIndexDeletion +import com.ritense.document.opensearch.repository.OpenSearchReconcileStateRepository +import com.ritense.document.opensearch.repository.PendingIndexDeletionRepository +import com.ritense.document.repository.impl.JsonSchemaDocumentRepository +import net.javacrumbs.shedlock.core.LockConfiguration +import net.javacrumbs.shedlock.core.LockProvider +import org.assertj.core.api.Assertions.assertThat +import org.junit.jupiter.api.AfterEach +import org.junit.jupiter.api.Test +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.security.test.context.support.WithMockUser +import org.springframework.transaction.PlatformTransactionManager +import org.springframework.transaction.annotation.Propagation +import org.springframework.transaction.annotation.Transactional +import org.springframework.transaction.support.TransactionTemplate +import java.time.Duration +import java.time.Instant +import java.time.LocalDateTime +import java.util.UUID + +/** + * Integration tests for [DocumentOpenSearchReconcileService] against a real PostgreSQL + OpenSearch. + * Runs **non-transactionally** so `changed_on`, the watermark state and the pending-index-deletion table + * behave as in production (own short transactions, committed data). Each test seeds the watermark explicitly. + */ +@Transactional(propagation = Propagation.NOT_SUPPORTED) +@WithMockUser(username = BaseOpenSearchIntegrationTest.USERNAME, authorities = [BaseOpenSearchIntegrationTest.FULL_ACCESS_ROLE]) +class DocumentOpenSearchReconcileIntTest : BaseOpenSearchIntegrationTest() { + + @Autowired + lateinit var reconcileService: DocumentOpenSearchReconcileService + + @Autowired + lateinit var syncService: DocumentOpenSearchSyncService + + @Autowired + lateinit var stateRepository: OpenSearchReconcileStateRepository + + @Autowired + lateinit var pendingIndexDeletionRepository: PendingIndexDeletionRepository + + @Autowired + lateinit var documentRepository: JsonSchemaDocumentRepository + + @Autowired + lateinit var transactionManager: PlatformTransactionManager + + @Autowired + lateinit var lockProvider: LockProvider + + @AfterEach + fun cleanUp() { + runWithoutAuthorization { documentService.removeDocuments("house") } + clearIndex() + stateRepository.deleteAll() + pendingIndexDeletionRepository.deleteAll() + } + + @Test + fun `reconcile indexes documents changed since the watermark and advances it`() { + val document = createDocument("reconcile-me") + clearIndex() + seedWatermark(LocalDateTime.now().minusHours(1)) + + reconcileService.reconcile() + + refreshIndex() + assertThat(openSearchRepository.findById(document.id().toString())).isPresent + val watermark = stateRepository.findById(OpenSearchReconcileState.SINGLETON_ID).get().watermark + assertThat(watermark).isAfter(LocalDateTime.now().minusMinutes(30)) + } + + @Test + fun `reconcile picks up a status change that has no live event and no modifiedOn bump`() { + val document = createDocument("status-doc") + runWithoutAuthorization { documentService.setInternalStatus(document.id(), "started") } + clearIndex() + seedWatermark(LocalDateTime.now().minusHours(1)) + + reconcileService.reconcile() + + refreshIndex() + val indexed = openSearchRepository.findById(document.id().toString()) + assertThat(indexed).isPresent + assertThat(indexed.get().internalStatus).isEqualTo("started") + } + + @Test + fun `reconcile is skipped while another writer holds the ShedLock`() { + val document = createDocument("locked-out") + clearIndex() + seedWatermark(LocalDateTime.now().minusHours(1)) + + val lock = lockProvider.lock( + LockConfiguration( + Instant.now(), + DocumentOpenSearchReconcileService.LOCK_NAME, + Duration.ofMinutes(5), + Duration.ZERO, + ) + ) + assertThat(lock).isPresent + try { + reconcileService.reconcile() + } finally { + lock.get().unlock() + } + + refreshIndex() + assertThat(openSearchRepository.findById(document.id().toString())).isEmpty + } + + @Test + fun `reconcile drains pending index deletions, removing docs from the index`() { + val document = createDocument("to-delete") + val id = document.id().id + syncService.upsertById(id) + refreshIndex() + assertThat(openSearchRepository.findById(id.toString())).isPresent + + // Delete from PostgreSQL and record the pending deletion the in-transaction listener would have written. + runWithoutAuthorization { documentService.deleteDocument(document.id()) } + pendingIndexDeletionRepository.save(PendingIndexDeletion(documentId = id)) + seedWatermark(LocalDateTime.now().minusHours(1)) + + reconcileService.reconcile() + + refreshIndex() + assertThat(openSearchRepository.findById(id.toString())).isEmpty + assertThat(pendingIndexDeletionRepository.count()).isZero() + } + + @Test + fun `changed_on advances on a status change while modifiedOn stays unchanged`() { + val document = createDocument("changed-on-doc") + val createdChangedOn = readChangedOn(document.id().id) + val initialModifiedOn = readModifiedOn(document.id().id) + // DATETIME can be second-resolution on MySQL; sleep past a full second so the bump is observable. + Thread.sleep(1100) + + runWithoutAuthorization { documentService.setInternalStatus(document.id(), "started") } + + assertThat(readChangedOn(document.id().id)).isAfter(createdChangedOn) + assertThat(readModifiedOn(document.id().id)).isEqualTo(initialModifiedOn) + } + + private fun seedWatermark(watermark: LocalDateTime) { + stateRepository.save(OpenSearchReconcileState(watermark = watermark)) + } + + private fun readChangedOn(id: UUID): LocalDateTime = + TransactionTemplate(transactionManager).execute { + documentRepository.findById(JsonSchemaDocumentId.existingId(id)).get().changedOn() + }!! + + private fun readModifiedOn(id: UUID) = + TransactionTemplate(transactionManager).execute { + documentRepository.findById(JsonSchemaDocumentId.existingId(id)).get().modifiedOn() + }!! + + private fun clearIndex() { + openSearchRepository.deleteAll() + refreshIndex() + } + + private fun createDocument(street: String): JsonSchemaDocument = + runWithoutAuthorization { + documentService.createDocument( + NewDocumentRequest("house", "house", "1.0.0", objectMapper.createObjectNode().put("street", street)) + ).resultingDocument().get() + } +} diff --git a/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchReindexServiceIntTest.kt b/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchReindexServiceIntTest.kt new file mode 100644 index 0000000000..0bde660a28 --- /dev/null +++ b/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchReindexServiceIntTest.kt @@ -0,0 +1,364 @@ +/* + * Copyright 2015-2026 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.ritense.document.opensearch.service + +import com.ritense.authorization.AuthorizationContext.Companion.runWithoutAuthorization +import com.ritense.document.domain.impl.JsonSchemaDocument +import com.ritense.document.domain.impl.JsonSchemaDocumentId +import com.ritense.document.domain.impl.request.NewDocumentRequest +import com.ritense.document.opensearch.BaseOpenSearchIntegrationTest +import com.ritense.document.opensearch.domain.OpenSearchReindexRun +import com.ritense.document.opensearch.domain.ReindexRunStatus +import com.ritense.document.opensearch.repository.OpenSearchReindexRunRepository +import jakarta.persistence.EntityManager +import net.javacrumbs.shedlock.core.LockConfiguration +import net.javacrumbs.shedlock.core.LockProvider +import org.assertj.core.api.Assertions.assertThat +import org.junit.jupiter.api.AfterEach +import org.junit.jupiter.api.Test +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.security.test.context.support.WithMockUser +import org.springframework.transaction.PlatformTransactionManager +import org.springframework.transaction.annotation.Propagation +import org.springframework.transaction.annotation.Transactional +import org.springframework.transaction.support.TransactionTemplate +import java.time.Duration +import java.time.Instant +import java.time.LocalDateTime +import java.util.UUID + +/** + * Integration tests for [DocumentOpenSearchReindexService]. + * + * These tests run **non-transactionally** ([Propagation.NOT_SUPPORTED], overriding the base's + * `@Transactional`). The production re-index runs on a background executor with no ambient transaction, + * committing its run-state updates in their own short transactions; reproducing that here is the only way + * the keyset reads and the persisted progress/cursor behave as they do in production. Committed test data + * is removed in [cleanUp]. + * + * Because the tests commit, creating a document can also trigger the live event sync + * ([com.ritense.document.opensearch.handler.DocumentOpenSearchEventListener]) which indexes that document + * into OpenSearch. To assert on the re-index in isolation we [clearIndex] after the document setup and + * before running the re-index, so the index reflects only what the re-index (re)indexed. + */ +@Transactional(propagation = Propagation.NOT_SUPPORTED) +@WithMockUser(username = BaseOpenSearchIntegrationTest.USERNAME, authorities = [BaseOpenSearchIntegrationTest.FULL_ACCESS_ROLE]) +class DocumentOpenSearchReindexServiceIntTest : BaseOpenSearchIntegrationTest() { + + @Autowired + lateinit var reindexService: DocumentOpenSearchReindexService + + @Autowired + lateinit var reindexRunService: OpenSearchReindexRunService + + @Autowired + lateinit var reindexRunRepository: OpenSearchReindexRunRepository + + @Autowired + lateinit var lockProvider: LockProvider + + @Autowired + lateinit var entityManager: EntityManager + + @Autowired + lateinit var transactionManager: PlatformTransactionManager + + @Autowired + lateinit var converter: JsonSchemaDocumentOsConverter + + @AfterEach + fun cleanUp() { + runWithoutAuthorization { documentService.removeDocuments("house") } + reindexRunRepository.deleteAll() + } + + @Test + fun `full re-index indexes every document and completes`() { + val ids = (1..5).map { createDocument("street-$it").id().id } + clearIndex() + + val (runId, processed) = reindex(ReindexRequest()) + + assertThat(processed).isEqualTo(5L) + val run = reindexRunRepository.findById(runId).get() + assertThat(run.status).isEqualTo(ReindexRunStatus.COMPLETED) + assertThat(run.processedCount).isEqualTo(5L) + refreshIndex() + assertThat(openSearchRepository.count()).isEqualTo(5L) + ids.forEach { assertThat(openSearchRepository.findById(it.toString())).isPresent } + } + + @Test + fun `re-index populates internalStatus from the live entity (C1)`() { + val document = createDocument("with-status") + runWithoutAuthorization { documentService.setInternalStatus(document.id(), "started") } + clearIndex() + + reindex(ReindexRequest()) + + refreshIndex() + val indexed = openSearchRepository.findById(document.id().toString()) + assertThat(indexed).isPresent + assertThat(indexed.get().internalStatus).isEqualTo("started") + } + + @Test + fun `scoped re-index by documentDefinitionName only indexes matching documents`() { + createDocument("house-doc-1") + createDocument("house-doc-2") + clearIndex() + + reindex(ReindexRequest(documentDefinitionName = "house")) + refreshIndex() + assertThat(openSearchRepository.count()).isEqualTo(2L) + + clearIndex() + + val (_, processed) = reindex(ReindexRequest(documentDefinitionName = "does-not-exist")) + assertThat(processed).isEqualTo(0L) + refreshIndex() + assertThat(openSearchRepository.count()).isEqualTo(0L) + } + + @Test + fun `scoped re-index by documentIds only indexes the requested subset`() { + val target = createDocument("target") + createDocument("other-1") + createDocument("other-2") + clearIndex() + + val (_, processed) = reindex(ReindexRequest(documentIds = listOf(target.id().id))) + + assertThat(processed).isEqualTo(1L) + refreshIndex() + assertThat(openSearchRepository.count()).isEqualTo(1L) + assertThat(openSearchRepository.findById(target.id().toString())).isPresent + } + + @Test + fun `scoped re-index by modifiedAfter only indexes documents modified after the cutoff`() { + val untouched = createDocument("untouched") // modifiedOn stays null -> excluded + val modified = createDocument("before-modify") + val cutoff = LocalDateTime.now() + Thread.sleep(50) + runWithoutAuthorization { + documentService.modifyDocument(modified, objectMapper.createObjectNode().put("street", "after-modify")) + } + clearIndex() + + reindex(ReindexRequest(modifiedAfter = cutoff)) + + refreshIndex() + assertThat(openSearchRepository.findById(modified.id().toString())).isPresent + assertThat(openSearchRepository.findById(untouched.id().toString())).isEmpty + } + + @Test + fun `re-index resumes from the persisted cursor of a prior run`() { + (1..6).forEach { createDocument("doc-$it") } + // Use the database's own ascending id ordering (PostgreSQL orders UUIDs unsigned, which differs + // from Kotlin's signed UUID.compareTo) so the cursor and expected set match the keyset query. + val dbOrderedIds = TransactionTemplate(transactionManager).execute { + entityManager + .createQuery("SELECT d.id.id FROM JsonSchemaDocument d ORDER BY d.id.id", UUID::class.java) + .resultList + }!! + val cursor = dbOrderedIds[2] // resume after the 3rd id -> 3 docs remain + val expectedIds = dbOrderedIds.subList(3, dbOrderedIds.size) + val seededRun = reindexRunRepository.save( + OpenSearchReindexRun( + id = UUID.randomUUID(), + status = ReindexRunStatus.FAILED, + pageSize = ReindexRequest.DEFAULT_PAGE_SIZE, + lastId = cursor, + processedCount = 3, + ) + ) + clearIndex() + + reindexService.reindex(seededRun.id) + + refreshIndex() + assertThat(openSearchRepository.count()).isEqualTo(expectedIds.size.toLong()) + expectedIds.forEach { assertThat(openSearchRepository.findById(it.toString())).isPresent } + assertThat(reindexRunRepository.findById(seededRun.id).get().status).isEqualTo(ReindexRunStatus.COMPLETED) + } + + @Test + fun `startup reconciliation marks a stale-heartbeat orphaned RUNNING run as FAILED`() { + val orphan = reindexRunRepository.save( + OpenSearchReindexRun( + id = UUID.randomUUID(), + status = ReindexRunStatus.RUNNING, + pageSize = ReindexRequest.DEFAULT_PAGE_SIZE, + heartbeatOn = LocalDateTime.now().minusHours(1), + ) + ) + + reindexRunService.reconcileOrphanedRuns() + + assertThat(reindexRunRepository.findById(orphan.id).get().status).isEqualTo(ReindexRunStatus.FAILED) + } + + @Test + fun `start returns null when the cluster-wide lock is already held`() { + val lock = lockProvider.lock( + LockConfiguration( + Instant.now(), + DocumentOpenSearchReindexService.LOCK_NAME, + Duration.ofMinutes(5), + Duration.ZERO, + ) + ) + assertThat(lock).isPresent + try { + assertThat(reindexService.start(ReindexRequest())).isNull() + } finally { + lock.get().unlock() + } + } + + @Test + fun `isReindexRunning is true while a running run has a fresh heartbeat`() { + reindexRunRepository.save( + OpenSearchReindexRun( + id = UUID.randomUUID(), + status = ReindexRunStatus.RUNNING, + pageSize = ReindexRequest.DEFAULT_PAGE_SIZE, + heartbeatOn = LocalDateTime.now(), + ) + ) + + assertThat(reindexRunService.isReindexRunning(Duration.ofMinutes(5))).isTrue() + } + + @Test + fun `isReindexRunning is false when the only running run has a stale heartbeat`() { + reindexRunRepository.save( + OpenSearchReindexRun( + id = UUID.randomUUID(), + status = ReindexRunStatus.RUNNING, + pageSize = ReindexRequest.DEFAULT_PAGE_SIZE, + heartbeatOn = LocalDateTime.now().minusMinutes(10), + ) + ) + + assertThat(reindexRunService.isReindexRunning(Duration.ofMinutes(5))).isFalse() + } + + @Test + fun `isReindexRunning is false when no run is RUNNING`() { + reindexRunRepository.save( + OpenSearchReindexRun( + id = UUID.randomUUID(), + status = ReindexRunStatus.COMPLETED, + pageSize = ReindexRequest.DEFAULT_PAGE_SIZE, + heartbeatOn = LocalDateTime.now(), + ) + ) + + assertThat(reindexRunService.isReindexRunning(Duration.ofMinutes(5))).isFalse() + } + + @Test + fun `reindex with pruneOrphans deletes orphans from OpenSearch`() { + val doc1 = createDocument("keep") + val doc2 = createDocument("orphan-1") + val doc3 = createDocument("orphan-2") + indexDocuments(doc1, doc2, doc3) + assertThat(openSearchRepository.count()).isEqualTo(3L) + + deleteDocumentFromDatabaseOnly(doc2.id()) + deleteDocumentFromDatabaseOnly(doc3.id()) + + val (runId, _) = reindex(ReindexRequest(pruneOrphans = true)) + + refreshIndex() + assertThat(openSearchRepository.count()).isEqualTo(1L) + assertThat(openSearchRepository.findById(doc1.id().toString())).isPresent + assertThat(openSearchRepository.findById(doc2.id().toString())).isEmpty + assertThat(openSearchRepository.findById(doc3.id().toString())).isEmpty + + val run = reindexRunRepository.findById(runId).get() + assertThat(run.prunedCount).isEqualTo(2L) + } + + @Test + fun `scoped reindex with pruneOrphans only prunes matching definition`() { + val houseDoc = createDocument("house-street") + indexDocuments(houseDoc) + + deleteDocumentFromDatabaseOnly(houseDoc.id()) + + reindex(ReindexRequest(documentDefinitionName = "house", pruneOrphans = true)) + + refreshIndex() + assertThat(openSearchRepository.findById(houseDoc.id().toString())).isEmpty + } + + @Test + fun `reindex without pruneOrphans leaves orphans in place`() { + val doc = createDocument("orphan") + indexDocuments(doc) + + deleteDocumentFromDatabaseOnly(doc.id()) + + reindex(ReindexRequest(pruneOrphans = false)) + + refreshIndex() + assertThat(openSearchRepository.findById(doc.id().toString())).isPresent + } + + private fun indexDocuments(vararg documents: JsonSchemaDocument) { + documents.forEach { doc -> + val osDoc = converter.toOsDocument(doc) + openSearchRepository.save(osDoc) + } + refreshIndex() + } + + private fun deleteDocumentFromDatabaseOnly(documentId: JsonSchemaDocumentId) { + TransactionTemplate(transactionManager).execute { + entityManager.createNativeQuery( + "DELETE FROM json_schema_document WHERE json_schema_document_id = :id" + ).setParameter("id", documentId.id).executeUpdate() + } + } + + /** Clears the OpenSearch index (and refreshes) so a subsequent assertion sees only the re-index output. */ + private fun clearIndex() { + openSearchRepository.deleteAll() + refreshIndex() + } + + /** + * Creates a run for [request] and drives the re-index synchronously (each step still uses its own + * transaction, since the test runs without an ambient one). Returns the run id and the documents + * processed. + */ + private fun reindex(request: ReindexRequest): Pair { + val run = reindexRunService.startOrResume(request) + return run.id to reindexService.reindex(run.id) + } + + private fun createDocument(street: String): JsonSchemaDocument = + runWithoutAuthorization { + documentService.createDocument( + NewDocumentRequest("house", "house", "1.0.0", objectMapper.createObjectNode().put("street", street)) + ).resultingDocument().get() + } +} diff --git a/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchReindexServiceTest.kt b/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchReindexServiceTest.kt new file mode 100644 index 0000000000..ce06e07198 --- /dev/null +++ b/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchReindexServiceTest.kt @@ -0,0 +1,112 @@ +/* + * Copyright 2015-2026 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.ritense.document.opensearch.service + +import com.ritense.document.opensearch.domain.OpenSearchReindexRun +import com.ritense.document.opensearch.domain.ReindexRunStatus +import com.ritense.document.opensearch.repository.JsonSchemaDocumentOpenSearchRepository +import jakarta.persistence.EntityManager +import net.javacrumbs.shedlock.core.LockProvider +import net.javacrumbs.shedlock.core.SimpleLock +import org.assertj.core.api.Assertions.assertThat +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test +import org.mockito.kotlin.any +import org.mockito.kotlin.mock +import org.mockito.kotlin.never +import org.mockito.kotlin.timeout +import org.mockito.kotlin.verify +import org.mockito.kotlin.whenever +import org.springframework.data.elasticsearch.core.ElasticsearchOperations +import org.springframework.transaction.PlatformTransactionManager +import java.util.Optional +import java.util.UUID + +class DocumentOpenSearchReindexServiceTest { + + private val entityManager: EntityManager = mock() + private val converter: JsonSchemaDocumentOsConverter = mock() + private val elasticsearchOperations: ElasticsearchOperations = mock() + private val transactionManager: PlatformTransactionManager = mock() + private val lockProvider: LockProvider = mock() + private val runService: OpenSearchReindexRunService = mock() + private val openSearchRepository: JsonSchemaDocumentOpenSearchRepository = mock() + + private lateinit var service: DocumentOpenSearchReindexService + + @BeforeEach + fun setUp() { + service = DocumentOpenSearchReindexService( + entityManager, + converter, + elasticsearchOperations, + transactionManager, + lockProvider, + runService, + openSearchRepository, + ) + } + + @Test + fun `start returns null and creates no run when the lock is already held`() { + whenever(lockProvider.lock(any())).thenReturn(Optional.empty()) + + val runId = service.start(ReindexRequest()) + + assertThat(runId).isNull() + verify(runService, never()).startOrResume(any()) + } + + @Test + fun `start creates a run and releases the lock when acquired`() { + val simpleLock: SimpleLock = mock() + whenever(lockProvider.lock(any())).thenReturn(Optional.of(simpleLock)) + val expectedId = UUID.randomUUID() + whenever(runService.startOrResume(any())).thenReturn(run(expectedId)) + + val runId = service.start(ReindexRequest()) + + assertThat(runId).isEqualTo(expectedId) + verify(runService).startOrResume(any()) + // The dispatched run finishes (here it terminates early against the mocks); the lock must be released. + verify(simpleLock, timeout(5_000)).unlock() + } + + @Test + fun `reindex marks the run STOPPED when cancellation was requested`() { + val runId = UUID.randomUUID() + whenever(runService.scopeOf(runId)).thenReturn(ReindexRequest()) + whenever(runService.pageSizeOf(runId)).thenReturn(ReindexRequest.DEFAULT_PAGE_SIZE) + whenever(runService.cursorOf(runId)).thenReturn(null) + whenever(runService.processedOf(runId)).thenReturn(0L) + + // destroy() sets the cancellation flag (and shuts down the idle executor). + service.destroy() + + val processed = service.reindex(runId) + + assertThat(processed).isEqualTo(0L) + verify(runService).stop(runId) + verify(runService, never()).complete(any(), any()) + } + + private fun run(id: UUID) = OpenSearchReindexRun( + id = id, + status = ReindexRunStatus.RUNNING, + pageSize = 100, + ) +} diff --git a/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchSyncServiceTest.kt b/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchSyncServiceTest.kt new file mode 100644 index 0000000000..b05e1639d7 --- /dev/null +++ b/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchSyncServiceTest.kt @@ -0,0 +1,128 @@ +/* + * Copyright 2015-2026 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.ritense.document.opensearch.service + +import com.ritense.document.domain.impl.JsonSchemaDocument +import com.ritense.document.domain.impl.JsonSchemaDocumentId +import com.ritense.document.opensearch.domain.JsonSchemaDocumentOsDocument +import com.ritense.document.opensearch.repository.JsonSchemaDocumentOpenSearchRepository +import com.ritense.document.repository.impl.JsonSchemaDocumentRepository +import org.assertj.core.api.Assertions.assertThatCode +import org.assertj.core.api.Assertions.assertThatThrownBy +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test +import org.mockito.kotlin.any +import org.mockito.kotlin.mock +import org.mockito.kotlin.never +import org.mockito.kotlin.verify +import org.mockito.kotlin.whenever +import org.springframework.data.elasticsearch.VersionConflictException +import org.springframework.transaction.PlatformTransactionManager +import org.springframework.transaction.support.SimpleTransactionStatus +import java.util.Optional +import java.util.UUID + +class DocumentOpenSearchSyncServiceTest { + + private val repository: JsonSchemaDocumentOpenSearchRepository = mock() + private val documentRepository: JsonSchemaDocumentRepository = mock() + private val converter: JsonSchemaDocumentOsConverter = mock() + private val transactionManager: PlatformTransactionManager = mock() + private lateinit var service: DocumentOpenSearchSyncService + + @BeforeEach + fun setUp() { + // Let the read-only TransactionTemplate run its callback inline. + whenever(transactionManager.getTransaction(any())).thenReturn(SimpleTransactionStatus()) + service = DocumentOpenSearchSyncService(repository, documentRepository, converter, transactionManager) + } + + @Test + fun `upsertById reloads the document and saves the converted os document`() { + val id = UUID.randomUUID() + val document: JsonSchemaDocument = mock() + val osDocument = osDocument(id.toString()) + whenever(documentRepository.findById(JsonSchemaDocumentId.existingId(id))).thenReturn(Optional.of(document)) + whenever(converter.toOsDocument(document)).thenReturn(osDocument) + + service.upsertById(id) + + verify(repository).save(osDocument) + } + + @Test + fun `upsertById swallows a version conflict (a newer version is already indexed)`() { + val id = UUID.randomUUID() + val document: JsonSchemaDocument = mock() + val osDocument = osDocument(id.toString()) + whenever(documentRepository.findById(JsonSchemaDocumentId.existingId(id))).thenReturn(Optional.of(document)) + whenever(converter.toOsDocument(document)).thenReturn(osDocument) + whenever(repository.save(osDocument)).thenThrow(VersionConflictException("conflict")) + + assertThatCode { service.upsertById(id) }.doesNotThrowAnyException() + } + + @Test + fun `upsertById propagates a non-conflict failure`() { + val id = UUID.randomUUID() + val document: JsonSchemaDocument = mock() + val osDocument = osDocument(id.toString()) + whenever(documentRepository.findById(JsonSchemaDocumentId.existingId(id))).thenReturn(Optional.of(document)) + whenever(converter.toOsDocument(document)).thenReturn(osDocument) + whenever(repository.save(osDocument)).thenThrow(RuntimeException("transport error")) + + assertThatThrownBy { service.upsertById(id) }.isInstanceOf(RuntimeException::class.java) + } + + @Test + fun `upsertById skips a document that no longer exists (already deleted)`() { + val id = UUID.randomUUID() + whenever(documentRepository.findById(JsonSchemaDocumentId.existingId(id))).thenReturn(Optional.empty()) + + service.upsertById(id) + + verify(converter, never()).toOsDocument(any()) + verify(repository, never()).save(any()) + } + + @Test + fun `delete removes the document from opensearch by id`() { + val id = UUID.randomUUID() + + service.delete(id) + + verify(repository).deleteById(id.toString()) + } + + private fun osDocument(id: String) = JsonSchemaDocumentOsDocument( + id = id, + content = null, + definitionId = null, + createdOn = null, + modifiedOn = null, + createdBy = null, + sequence = null, + version = null, + assigneeId = null, + assigneeFullName = null, + internalStatus = null, + caseTags = null, + relations = null, + relatedFiles = null, + retentionDate = null, + ) +} diff --git a/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchVersioningIntTest.kt b/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchVersioningIntTest.kt new file mode 100644 index 0000000000..25bd233fe6 --- /dev/null +++ b/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/service/DocumentOpenSearchVersioningIntTest.kt @@ -0,0 +1,145 @@ +/* + * Copyright 2015-2026 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.ritense.document.opensearch.service + +import com.ritense.authorization.AuthorizationContext.Companion.runWithoutAuthorization +import com.ritense.document.domain.impl.JsonSchemaDocument +import com.ritense.document.domain.impl.request.NewDocumentRequest +import com.ritense.document.opensearch.BaseOpenSearchIntegrationTest +import com.ritense.document.opensearch.domain.JsonSchemaDocumentOsDocument +import org.assertj.core.api.Assertions.assertThat +import org.assertj.core.api.Assertions.assertThatCode +import org.junit.jupiter.api.AfterEach +import org.junit.jupiter.api.Test +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.security.test.context.support.WithMockUser +import org.springframework.transaction.annotation.Propagation +import org.springframework.transaction.annotation.Transactional +import java.util.UUID + +/** + * Integration tests for OpenSearch external versioning against a real OpenSearch. "Highest version wins": + * a re-send of an equal-or-lower [JsonSchemaDocumentOsDocument.indexVersion] is a benign version-conflict + * no-op (swallowed by [JsonSchemaDocumentOsConverter.indexChunk] / [DocumentOpenSearchSyncService]) and can + * never overwrite a newer document; a strictly higher version updates. + * + * Runs **non-transactionally** so the live-sync path sees committed data and its own JPA `version` bumps. + */ +@Transactional(propagation = Propagation.NOT_SUPPORTED) +@WithMockUser(username = BaseOpenSearchIntegrationTest.USERNAME, authorities = [BaseOpenSearchIntegrationTest.FULL_ACCESS_ROLE]) +class DocumentOpenSearchVersioningIntTest : BaseOpenSearchIntegrationTest() { + + @Autowired + lateinit var converter: JsonSchemaDocumentOsConverter + + @Autowired + lateinit var syncService: DocumentOpenSearchSyncService + + @AfterEach + fun cleanUp() { + runWithoutAuthorization { documentService.removeDocuments("house") } + clearIndex() + } + + @Test + fun `re-indexing the same version is a benign no-op`() { + val id = UUID.randomUUID().toString() + openSearchRepository.saveAll(listOf(osDoc(id, indexVersion = 5, internalStatus = "first"))) + refreshIndex() + + val skipped = converter.indexChunk(listOf(osDoc(id, indexVersion = 5, internalStatus = "second"))) + refreshIndex() + + assertThat(skipped).isZero() + assertThat(openSearchRepository.findById(id).get().internalStatus).isEqualTo("first") + } + + @Test + fun `a stale lower-version write does not overwrite a newer document (order independence)`() { + val id = UUID.randomUUID().toString() + openSearchRepository.saveAll(listOf(osDoc(id, indexVersion = 6, internalStatus = "sixth"))) + refreshIndex() + + val skipped = converter.indexChunk(listOf(osDoc(id, indexVersion = 5, internalStatus = "fifth"))) + refreshIndex() + + assertThat(skipped).isZero() + assertThat(openSearchRepository.findById(id).get().internalStatus).isEqualTo("sixth") + } + + @Test + fun `a strictly higher version updates the document`() { + val id = UUID.randomUUID().toString() + openSearchRepository.saveAll(listOf(osDoc(id, indexVersion = 5, internalStatus = "old"))) + refreshIndex() + + val skipped = converter.indexChunk(listOf(osDoc(id, indexVersion = 6, internalStatus = "new"))) + refreshIndex() + + assertThat(skipped).isZero() + assertThat(openSearchRepository.findById(id).get().internalStatus).isEqualTo("new") + } + + @Test + fun `the live path indexes a real change and swallows a redundant re-send`() { + val document = createDocument("versioned") + val id = document.id().id + syncService.upsertById(id) + refreshIndex() + + runWithoutAuthorization { documentService.setInternalStatus(document.id(), "started") } + syncService.upsertById(id) + refreshIndex() + assertThat(openSearchRepository.findById(id.toString()).get().internalStatus).isEqualTo("started") + + // Re-sending the same (now-current) version is a version conflict — swallowed, no exception. + assertThatCode { syncService.upsertById(id) }.doesNotThrowAnyException() + refreshIndex() + assertThat(openSearchRepository.findById(id.toString()).get().internalStatus).isEqualTo("started") + } + + private fun clearIndex() { + openSearchRepository.deleteAll() + refreshIndex() + } + + private fun createDocument(street: String): JsonSchemaDocument = + runWithoutAuthorization { + documentService.createDocument( + NewDocumentRequest("house", "house", "1.0.0", objectMapper.createObjectNode().put("street", street)) + ).resultingDocument().get() + } + + private fun osDoc(id: String, indexVersion: Long, internalStatus: String) = JsonSchemaDocumentOsDocument( + id = id, + content = null, + definitionId = null, + createdOn = null, + modifiedOn = null, + createdBy = null, + sequence = null, + version = null, + assigneeId = null, + assigneeFullName = null, + internalStatus = internalStatus, + caseTags = null, + relations = null, + relatedFiles = null, + retentionDate = null, + indexVersion = indexVersion, + ) +} diff --git a/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/service/JsonSchemaDocumentOpenSearchServiceIntTest.kt b/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/service/JsonSchemaDocumentOpenSearchServiceIntTest.kt new file mode 100644 index 0000000000..cee7a81f7e --- /dev/null +++ b/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/service/JsonSchemaDocumentOpenSearchServiceIntTest.kt @@ -0,0 +1,154 @@ +/* + * Copyright 2015-2025 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.ritense.document.opensearch.service + +import com.ritense.authorization.AuthorizationContext.Companion.runWithoutAuthorization +import com.ritense.document.domain.impl.JsonSchemaDocument +import com.ritense.document.domain.impl.request.NewDocumentRequest +import com.ritense.document.domain.search.AdvancedSearchRequest +import com.ritense.document.opensearch.BaseOpenSearchIntegrationTest +import com.ritense.document.opensearch.domain.JsonSchemaDocumentOsDocument +import com.ritense.document.opensearch.domain.OsBlueprintId +import com.ritense.document.opensearch.domain.OsDefinitionId +import com.ritense.document.service.DocumentSearchService +import com.ritense.valtimo.contract.blueprint.BlueprintType +import org.assertj.core.api.Assertions.assertThat +import org.junit.jupiter.api.Test +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.data.domain.PageRequest +import org.springframework.security.test.context.support.WithMockUser + +@WithMockUser(username = BaseOpenSearchIntegrationTest.USERNAME, authorities = [BaseOpenSearchIntegrationTest.FULL_ACCESS_ROLE]) +class JsonSchemaDocumentOpenSearchServiceIntTest : BaseOpenSearchIntegrationTest() { + + @Autowired + lateinit var documentSearchService: DocumentSearchService + + @Test + fun `globalSearchFilter returns matching document`() { + seedDocument("Funenpark") + + val page = documentSearchService.search( + "house", + BlueprintType.CASE, + AdvancedSearchRequest().globalSearchFilter("Funenpark"), + PageRequest.of(0, 10) + ) + + assertThat(page.totalElements).isEqualTo(1L) + } + + @Test + fun `globalSearchFilter is case insensitive`() { + seedDocument("Funenpark") + + val page = documentSearchService.search( + "house", + BlueprintType.CASE, + AdvancedSearchRequest().globalSearchFilter("FUNENPARK"), + PageRequest.of(0, 10) + ) + + assertThat(page.totalElements).isEqualTo(1L) + } + + @Test + fun `globalSearchFilter excludes non-matching documents`() { + val docA = seedDocument("Funenpark") + seedDocument("Keizersgracht") + + val page = documentSearchService.search( + "house", + BlueprintType.CASE, + AdvancedSearchRequest().globalSearchFilter("Funenpark"), + PageRequest.of(0, 10) + ) + + assertThat(page.totalElements).isEqualTo(1L) + assertThat(page.content[0].id()).isEqualTo(docA.id()) + } + + @Test + fun `no globalSearchFilter returns all authorized documents`() { + seedDocument("Funenpark") + seedDocument("Keizersgracht") + + val page = documentSearchService.search( + "house", + BlueprintType.CASE, + AdvancedSearchRequest(), + PageRequest.of(0, 10) + ) + + assertThat(page.totalElements).isEqualTo(2L) + } + + @Test + fun `globalSearchFilter supports partial match`() { + seedDocument("Keizersgracht") + + val page = documentSearchService.search( + "house", + BlueprintType.CASE, + AdvancedSearchRequest().globalSearchFilter("Keizers"), + PageRequest.of(0, 10) + ) + + assertThat(page.totalElements).isEqualTo(1L) + } + + private fun seedDocument(street: String): JsonSchemaDocument { + val content = objectMapper.createObjectNode().apply { put("street", street) } + val jpaDoc = runWithoutAuthorization { + documentService.createDocument( + NewDocumentRequest("house", "house", "1.0.0", content) + ).resultingDocument().get() + } + openSearchRepository.save( + JsonSchemaDocumentOsDocument( + id = jpaDoc.id().toString(), + content = mapOf("street" to street), + definitionId = OsDefinitionId( + name = "house", + version = null, + blueprintId = OsBlueprintId( + blueprintType = "CASE", + blueprintKey = null, + blueprintVersionTag = null, + isBuildingBlock = null, + isCase = null, + ), + ), + createdOn = null, + modifiedOn = null, + createdBy = null, + sequence = null, + version = null, + assigneeId = null, + assigneeFullName = null, + internalStatus = null, + caseTags = null, + relations = null, + relatedFiles = null, + retentionDate = null, + contentText = street, + ) + ) + elasticsearchOperations.indexOps(JsonSchemaDocumentOsDocument::class.java).refresh() + return jpaDoc + } +} diff --git a/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/service/JsonSchemaDocumentOpenSearchServiceTest.kt b/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/service/JsonSchemaDocumentOpenSearchServiceTest.kt new file mode 100644 index 0000000000..0be2aa3a6d --- /dev/null +++ b/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/service/JsonSchemaDocumentOpenSearchServiceTest.kt @@ -0,0 +1,356 @@ +/* + * Copyright 2015-2026 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.ritense.document.opensearch.service + +import com.fasterxml.jackson.databind.ObjectMapper +import com.ritense.authorization.Action +import com.ritense.authorization.AuthorizationService +import com.ritense.case_.domain.definition.CaseDefinition +import com.ritense.case.service.CaseDefinitionService +import com.ritense.valtimo.contract.case_.CaseDefinitionId +import com.ritense.authorization.permission.ConditionContainer +import com.ritense.authorization.permission.Permission +import com.ritense.authorization.role.Role +import com.ritense.document.domain.impl.JsonSchemaDocument +import com.ritense.document.domain.impl.searchfield.SearchField +import com.ritense.document.domain.impl.searchfield.SearchFieldDataType +import com.ritense.document.domain.impl.searchfield.SearchFieldFieldType +import com.ritense.document.domain.impl.searchfield.SearchFieldMatchType +import com.ritense.document.domain.search.AdvancedSearchRequest +import com.ritense.document.opensearch.authorization.OpenSearchAuthorizationEntityMapper +import com.ritense.document.opensearch.authorization.OpenSearchPermissionConditionTranslator +import com.ritense.document.opensearch.domain.JsonSchemaDocumentOsDocument +import com.ritense.document.repository.impl.JsonSchemaDocumentRepository +import com.ritense.document.service.JsonSchemaDocumentActionProvider +import com.ritense.document.service.SearchFieldService +import com.ritense.document.service.impl.SearchRequest +import com.ritense.outbox.OutboxService +import com.ritense.valtimo.contract.authentication.UserManagementService +import com.ritense.valtimo.contract.blueprint.BlueprintType +import org.assertj.core.api.Assertions.assertThat +import org.junit.jupiter.api.AfterEach +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test +import org.mockito.kotlin.any +import org.mockito.kotlin.argumentCaptor +import org.mockito.kotlin.eq +import org.mockito.kotlin.mock +import org.mockito.kotlin.whenever +import org.springframework.data.domain.PageRequest +import org.springframework.data.elasticsearch.core.ElasticsearchOperations +import org.springframework.data.elasticsearch.core.SearchHits +import org.springframework.data.elasticsearch.core.query.StringQuery +import org.springframework.security.authentication.UsernamePasswordAuthenticationToken +import org.springframework.security.core.authority.SimpleGrantedAuthority +import org.springframework.security.core.context.SecurityContextHolder + +class JsonSchemaDocumentOpenSearchServiceTest { + + private val elasticsearchOperations: ElasticsearchOperations = mock() + private val authorizationService: AuthorizationService = mock() + private val jpaRepository: JsonSchemaDocumentRepository = mock() + private val userManagementService: UserManagementService = mock() + private val searchFieldService: SearchFieldService = mock() + private val outboxService: OutboxService = mock() + private val objectMapper: ObjectMapper = ObjectMapper() + private val caseDefinitionService: CaseDefinitionService = mock() + + private lateinit var service: JsonSchemaDocumentOpenSearchService + + @BeforeEach + fun setUp() { + val translator = OpenSearchPermissionConditionTranslator( + openSearchMappers = emptyList>(), + authorizationService = authorizationService, + documentRepository = jpaRepository, + ) + service = JsonSchemaDocumentOpenSearchService( + elasticsearchOperations = elasticsearchOperations, + translator = translator, + authorizationService = authorizationService, + jpaRepository = jpaRepository, + userManagementService = userManagementService, + searchFieldService = searchFieldService, + outboxService = outboxService, + objectMapper = objectMapper, + caseDefinitionService = caseDefinitionService, + ) + + val auth = UsernamePasswordAuthenticationToken( + USERNAME, + null, + listOf(SimpleGrantedAuthority(FULL_ACCESS_ROLE)), + ) + SecurityContextHolder.getContext().authentication = auth + + val role = Role(key = FULL_ACCESS_ROLE) + val viewListPermission = Permission( + resourceType = JsonSchemaDocument::class.java, + actions = mutableListOf(JsonSchemaDocumentActionProvider.VIEW_LIST), + conditionContainer = ConditionContainer(emptyList()), + role = role, + ) + whenever( + authorizationService.getPermissions( + eq(JsonSchemaDocument::class.java), + eq(JsonSchemaDocumentActionProvider.VIEW_LIST), + ) + ).thenReturn(listOf(viewListPermission)) + + val emptySearchHits: SearchHits = mock() + whenever(emptySearchHits.searchHits).thenReturn(emptyList()) + whenever(emptySearchHits.totalHits).thenReturn(0L) + whenever(elasticsearchOperations.search(any(), eq(JsonSchemaDocumentOsDocument::class.java))).thenReturn(emptySearchHits) + whenever(jpaRepository.findAllById(any())).thenReturn(emptyList()) + } + + @AfterEach + fun tearDown() { + SecurityContextHolder.clearContext() + } + + @Test + fun `search with globalSearchFilter and no search fields returns match none`() { + val queryCaptor = argumentCaptor() + val emptySearchHits: SearchHits = mock() + whenever(emptySearchHits.searchHits).thenReturn(emptyList()) + whenever(emptySearchHits.totalHits).thenReturn(0L) + whenever(elasticsearchOperations.search(queryCaptor.capture(), eq(JsonSchemaDocumentOsDocument::class.java))).thenReturn(emptySearchHits) + + val request = AdvancedSearchRequest().globalSearchFilter("Amsterdam") + service.search("house", BlueprintType.CASE, request, PageRequest.of(0, 10)) + + val capturedQuery = queryCaptor.firstValue + assertThat(capturedQuery.source).contains("must_not") + assertThat(capturedQuery.source).contains("match_all") + } + + @Test + fun `search without globalSearchFilter does not include contentText in query`() { + val queryCaptor = argumentCaptor() + val emptySearchHits: SearchHits = mock() + whenever(emptySearchHits.searchHits).thenReturn(emptyList()) + whenever(emptySearchHits.totalHits).thenReturn(0L) + whenever(elasticsearchOperations.search(queryCaptor.capture(), eq(JsonSchemaDocumentOsDocument::class.java))).thenReturn(emptySearchHits) + + val request = AdvancedSearchRequest() + service.search("house", BlueprintType.CASE, request, PageRequest.of(0, 10)) + + val capturedQuery = queryCaptor.firstValue + assertThat(capturedQuery.source).doesNotContain("contentText") + } + + @Test + fun `search with empty globalSearchFilter does not include contentText in query`() { + val queryCaptor = argumentCaptor() + val emptySearchHits: SearchHits = mock() + whenever(emptySearchHits.searchHits).thenReturn(emptyList()) + whenever(emptySearchHits.totalHits).thenReturn(0L) + whenever(elasticsearchOperations.search(queryCaptor.capture(), eq(JsonSchemaDocumentOsDocument::class.java))).thenReturn(emptySearchHits) + + val request = AdvancedSearchRequest().globalSearchFilter("") + service.search("house", BlueprintType.CASE, request, PageRequest.of(0, 10)) + + val capturedQuery = queryCaptor.firstValue + assertThat(capturedQuery.source).doesNotContain("contentText") + } + + @Test + fun `search result uses count from opensearch`() { + val searchHits: SearchHits = mock() + whenever(searchHits.searchHits).thenReturn(emptyList()) + whenever(searchHits.totalHits).thenReturn(5L) + whenever(elasticsearchOperations.search(any(), eq(JsonSchemaDocumentOsDocument::class.java))).thenReturn(searchHits) + + val request = AdvancedSearchRequest().globalSearchFilter("test") + val page = service.search("house", BlueprintType.CASE, request, PageRequest.of(0, 10)) + + assertThat(page.totalElements).isEqualTo(5L) + } + + @Test + fun `search with field-qualified term targets specific field`() { + val queryCaptor = argumentCaptor() + val emptySearchHits: SearchHits = mock() + whenever(emptySearchHits.searchHits).thenReturn(emptyList()) + whenever(emptySearchHits.totalHits).thenReturn(0L) + whenever(elasticsearchOperations.search(queryCaptor.capture(), eq(JsonSchemaDocumentOsDocument::class.java))).thenReturn(emptySearchHits) + whenever(searchFieldService.getSearchFields("house")).thenReturn(listOf( + SearchField("city", "doc:city", SearchFieldDataType.TEXT, SearchFieldFieldType.SINGLE, SearchFieldMatchType.LIKE, null, 0, "City") + )) + + val request = AdvancedSearchRequest().globalSearchFilter("city:amsterdam") + service.search("house", BlueprintType.CASE, request, PageRequest.of(0, 10)) + + val capturedQuery = queryCaptor.firstValue + assertThat(capturedQuery.source).contains("content.city") + assertThat(capturedQuery.source).contains("amsterdam") + } + + @Test + fun `search with EXACT match type field does not add wildcards`() { + val queryCaptor = argumentCaptor() + val emptySearchHits: SearchHits = mock() + whenever(emptySearchHits.searchHits).thenReturn(emptyList()) + whenever(emptySearchHits.totalHits).thenReturn(0L) + whenever(elasticsearchOperations.search(queryCaptor.capture(), eq(JsonSchemaDocumentOsDocument::class.java))).thenReturn(emptySearchHits) + whenever(searchFieldService.getSearchFields("house")).thenReturn(listOf( + SearchField("status", "doc:status", SearchFieldDataType.TEXT, SearchFieldFieldType.SINGLE, SearchFieldMatchType.EXACT, null, 0, "Status") + )) + + val request = AdvancedSearchRequest().globalSearchFilter("status:active") + service.search("house", BlueprintType.CASE, request, PageRequest.of(0, 10)) + + val capturedQuery = queryCaptor.firstValue + assertThat(capturedQuery.source).contains("content.status:active") + assertThat(capturedQuery.source).doesNotContain("*active*") + } + + @Test + fun `search with LIKE match type field adds wildcards`() { + val queryCaptor = argumentCaptor() + val emptySearchHits: SearchHits = mock() + whenever(emptySearchHits.searchHits).thenReturn(emptyList()) + whenever(emptySearchHits.totalHits).thenReturn(0L) + whenever(elasticsearchOperations.search(queryCaptor.capture(), eq(JsonSchemaDocumentOsDocument::class.java))).thenReturn(emptySearchHits) + whenever(searchFieldService.getSearchFields("house")).thenReturn(listOf( + SearchField("name", "doc:name", SearchFieldDataType.TEXT, SearchFieldFieldType.SINGLE, SearchFieldMatchType.LIKE, null, 0, "Name") + )) + + val request = AdvancedSearchRequest().globalSearchFilter("name:john") + service.search("house", BlueprintType.CASE, request, PageRequest.of(0, 10)) + + val capturedQuery = queryCaptor.firstValue + assertThat(capturedQuery.source).contains("content.name:*john*") + } + + @Test + fun `search with quoted field value does not add wildcards even for LIKE fields`() { + val queryCaptor = argumentCaptor() + val emptySearchHits: SearchHits = mock() + whenever(emptySearchHits.searchHits).thenReturn(emptyList()) + whenever(emptySearchHits.totalHits).thenReturn(0L) + whenever(elasticsearchOperations.search(queryCaptor.capture(), eq(JsonSchemaDocumentOsDocument::class.java))).thenReturn(emptySearchHits) + whenever(searchFieldService.getSearchFields("house")).thenReturn(listOf( + SearchField("address", "doc:address", SearchFieldDataType.TEXT, SearchFieldFieldType.SINGLE, SearchFieldMatchType.LIKE, null, 0, "Address") + )) + + val request = AdvancedSearchRequest().globalSearchFilter("""address:"Main Street 123"""") + service.search("house", BlueprintType.CASE, request, PageRequest.of(0, 10)) + + val capturedQuery = queryCaptor.firstValue + assertThat(capturedQuery.source).contains("""content.address:\"Main Street 123\"""") + assertThat(capturedQuery.source).doesNotContain("*Main Street 123*") + } + + @Test + fun `search with unknown field throws exception listing unknown fields`() { + whenever(searchFieldService.getSearchFields("house")).thenReturn(listOf( + SearchField("city", "doc:city", SearchFieldDataType.TEXT, SearchFieldFieldType.SINGLE, SearchFieldMatchType.LIKE, null, 0, "City") + )) + + val request = AdvancedSearchRequest().globalSearchFilter("unknownField:value anotherBad:x city:amsterdam") + + val exception = org.junit.jupiter.api.assertThrows { + service.search("house", BlueprintType.CASE, request, PageRequest.of(0, 10)) + } + + assertThat(exception.message).contains("unknownField") + assertThat(exception.message).contains("anotherBad") + } + + @Test + fun `search with mixed qualified and unqualified terms`() { + val queryCaptor = argumentCaptor() + val emptySearchHits: SearchHits = mock() + whenever(emptySearchHits.searchHits).thenReturn(emptyList()) + whenever(emptySearchHits.totalHits).thenReturn(0L) + whenever(elasticsearchOperations.search(queryCaptor.capture(), eq(JsonSchemaDocumentOsDocument::class.java))).thenReturn(emptySearchHits) + whenever(searchFieldService.getSearchFields("house")).thenReturn(listOf( + SearchField("city", "doc:city", SearchFieldDataType.TEXT, SearchFieldFieldType.SINGLE, SearchFieldMatchType.EXACT, null, 0, "City") + )) + + val request = AdvancedSearchRequest().globalSearchFilter("city:amsterdam urgent") + service.search("house", BlueprintType.CASE, request, PageRequest.of(0, 10)) + + val capturedQuery = queryCaptor.firstValue + assertThat(capturedQuery.source).contains("content.city:amsterdam") + assertThat(capturedQuery.source).contains("*urgent*") + } + + @Test + fun `global search without definition name builds per-definition scoped query`() { + val queryCaptor = argumentCaptor() + val emptySearchHits: SearchHits = mock() + whenever(emptySearchHits.searchHits).thenReturn(emptyList()) + whenever(emptySearchHits.totalHits).thenReturn(0L) + whenever(elasticsearchOperations.search(queryCaptor.capture(), eq(JsonSchemaDocumentOsDocument::class.java))).thenReturn(emptySearchHits) + + val houseDef = mock() + val houseDefId = mock() + whenever(houseDefId.key).thenReturn("house") + whenever(houseDef.id).thenReturn(houseDefId) + + val carDef = mock() + val carDefId = mock() + whenever(carDefId.key).thenReturn("car") + whenever(carDef.id).thenReturn(carDefId) + + whenever(caseDefinitionService.getCaseDefinitions(active = true)).thenReturn(listOf(houseDef, carDef)) + whenever(searchFieldService.getSearchFields("house")).thenReturn(listOf( + SearchField("city", "doc:city", SearchFieldDataType.TEXT, SearchFieldFieldType.SINGLE, SearchFieldMatchType.LIKE, null, 0, "City") + )) + whenever(searchFieldService.getSearchFields("car")).thenReturn(listOf( + SearchField("brand", "doc:brand", SearchFieldDataType.TEXT, SearchFieldFieldType.SINGLE, SearchFieldMatchType.LIKE, null, 0, "Brand") + )) + + val request = SearchRequest() + request.globalSearchFilter = "test" + service.search(request, BlueprintType.CASE, PageRequest.of(0, 10)) + + val capturedQuery = queryCaptor.firstValue + assertThat(capturedQuery.source).contains("definitionId.name") + assertThat(capturedQuery.source).contains("house") + assertThat(capturedQuery.source).contains("car") + assertThat(capturedQuery.source).contains("content.city") + assertThat(capturedQuery.source).contains("content.brand") + } + + @Test + fun `global search without definition name and no accessible definitions adds matchNone query`() { + val queryCaptor = argumentCaptor() + val emptySearchHits: SearchHits = mock() + whenever(emptySearchHits.searchHits).thenReturn(emptyList()) + whenever(emptySearchHits.totalHits).thenReturn(0L) + whenever(elasticsearchOperations.search(queryCaptor.capture(), eq(JsonSchemaDocumentOsDocument::class.java))).thenReturn(emptySearchHits) + + whenever(caseDefinitionService.getCaseDefinitions(active = true)).thenReturn(emptyList()) + + val request = SearchRequest() + request.globalSearchFilter = "test" + service.search(request, BlueprintType.CASE, PageRequest.of(0, 10)) + + val capturedQuery = queryCaptor.firstValue + assertThat(capturedQuery.source).contains("must_not") + assertThat(capturedQuery.source).contains("match_all") + } + + companion object { + private const val FULL_ACCESS_ROLE = "full access role" + private const val USERNAME = "test@test.com" + } +} diff --git a/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/service/JsonSchemaDocumentOsConverterTest.kt b/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/service/JsonSchemaDocumentOsConverterTest.kt new file mode 100644 index 0000000000..48666c9d9e --- /dev/null +++ b/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/service/JsonSchemaDocumentOsConverterTest.kt @@ -0,0 +1,127 @@ +/* + * Copyright 2015-2026 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.ritense.document.opensearch.service + +import com.fasterxml.jackson.databind.ObjectMapper +import com.ritense.document.opensearch.domain.JsonSchemaDocumentOsDocument +import com.ritense.document.opensearch.repository.JsonSchemaDocumentOpenSearchRepository +import org.assertj.core.api.Assertions.assertThat +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test +import org.mockito.kotlin.any +import org.mockito.kotlin.eq +import org.mockito.kotlin.mock +import org.mockito.kotlin.never +import org.mockito.kotlin.verify +import org.mockito.kotlin.whenever +import org.springframework.data.elasticsearch.BulkFailureException +import org.springframework.data.elasticsearch.BulkFailureException.FailureDetails +import org.springframework.data.elasticsearch.VersionConflictException + +class JsonSchemaDocumentOsConverterTest { + + private val objectMapper: ObjectMapper = mock() + private val repository: JsonSchemaDocumentOpenSearchRepository = mock() + private lateinit var converter: JsonSchemaDocumentOsConverter + + @BeforeEach + fun setUp() { + converter = JsonSchemaDocumentOsConverter(objectMapper, repository) + } + + @Test + fun `indexChunk bulk-saves and reports zero skips on success`() { + val chunk = listOf(osDocument("a"), osDocument("b")) + + val skipped = converter.indexChunk(chunk) + + assertThat(skipped).isZero() + verify(repository).saveAll(chunk) + } + + @Test + fun `indexChunk re-processes only the failed documents and isolates a real failure`() { + val good = osDocument("good") + val poison = osDocument("poison") + whenever(repository.saveAll(any>())) + .thenThrow(BulkFailureException("bulk failed", mapOf("poison" to FailureDetails(400, "mapper_parsing_exception")))) + whenever(repository.save(eq(poison))).thenThrow(RuntimeException("mapping error")) + + val skipped = converter.indexChunk(listOf(good, poison)) + + assertThat(skipped).isEqualTo(1L) + verify(repository).save(poison) + // "good" was not in the failure map, so it is never re-processed. + verify(repository, never()).save(good) + } + + @Test + fun `indexChunk treats a version conflict as benign (no skip, no retry)`() { + val document = osDocument("a") + whenever(repository.saveAll(any>())) + .thenThrow(BulkFailureException("bulk failed", mapOf("a" to FailureDetails(409, "version_conflict_engine_exception ...")))) + + val skipped = converter.indexChunk(listOf(document)) + + assertThat(skipped).isZero() + // A 409 means the stored doc is already ≥ this version — never re-saved. + verify(repository, never()).save(any()) + } + + @Test + fun `indexChunk classifies a version conflict by message when status is absent`() { + val document = osDocument("a") + whenever(repository.saveAll(any>())) + .thenThrow(BulkFailureException("bulk failed", mapOf("a" to FailureDetails(null, "... version_conflict_engine_exception ...")))) + + val skipped = converter.indexChunk(listOf(document)) + + assertThat(skipped).isZero() + verify(repository, never()).save(any()) + } + + @Test + fun `indexChunk treats a VersionConflictException on retry as benign`() { + val document = osDocument("a") + whenever(repository.saveAll(any>())) + .thenThrow(BulkFailureException("bulk failed", mapOf("a" to FailureDetails(500, "transient")))) + whenever(repository.save(eq(document))).thenThrow(VersionConflictException("conflict")) + + val skipped = converter.indexChunk(listOf(document)) + + assertThat(skipped).isZero() + verify(repository).save(document) + } + + private fun osDocument(id: String) = JsonSchemaDocumentOsDocument( + id = id, + content = null, + definitionId = null, + createdOn = null, + modifiedOn = null, + createdBy = null, + sequence = null, + version = null, + assigneeId = null, + assigneeFullName = null, + internalStatus = null, + caseTags = null, + relations = null, + relatedFiles = null, + retentionDate = null, + ) +} diff --git a/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/service/OpenSearchReindexRunServiceTest.kt b/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/service/OpenSearchReindexRunServiceTest.kt new file mode 100644 index 0000000000..e26d1658c9 --- /dev/null +++ b/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/service/OpenSearchReindexRunServiceTest.kt @@ -0,0 +1,197 @@ +/* + * Copyright 2015-2026 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.ritense.document.opensearch.service + +import com.fasterxml.jackson.databind.ObjectMapper +import com.ritense.document.opensearch.OpenSearchProperties +import com.ritense.document.opensearch.domain.OpenSearchReindexRun +import com.ritense.document.opensearch.domain.ReindexRunStatus +import com.ritense.document.opensearch.repository.OpenSearchReindexRunRepository +import jakarta.persistence.EntityManager +import jakarta.persistence.TypedQuery +import jakarta.persistence.criteria.CriteriaBuilder +import jakarta.persistence.criteria.CriteriaQuery +import jakarta.persistence.criteria.Root +import org.assertj.core.api.Assertions.assertThat +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test +import org.mockito.kotlin.any +import org.mockito.kotlin.argumentCaptor +import org.mockito.kotlin.doAnswer +import org.mockito.kotlin.eq +import org.mockito.kotlin.mock +import org.mockito.kotlin.never +import org.mockito.kotlin.verify +import org.mockito.kotlin.whenever +import java.util.Optional +import java.util.UUID + +class OpenSearchReindexRunServiceTest { + + private val repository: OpenSearchReindexRunRepository = mock() + private val objectMapper: ObjectMapper = ObjectMapper().findAndRegisterModules() + private val properties = OpenSearchProperties() + private val entityManager: EntityManager = mock() + private lateinit var service: OpenSearchReindexRunService + + @BeforeEach + fun setUp() { + service = OpenSearchReindexRunService(repository, objectMapper, properties, entityManager) + whenever(repository.save(any())).doAnswer { it.arguments[0] as OpenSearchReindexRun } + setupEntityManagerMock() + } + + private fun setupEntityManagerMock() { + val criteriaBuilder: CriteriaBuilder = mock() + val criteriaQuery: CriteriaQuery = mock() + val root: Root<*> = mock() + val typedQuery: TypedQuery = mock() + + whenever(entityManager.criteriaBuilder).thenReturn(criteriaBuilder) + whenever(criteriaBuilder.createQuery(Long::class.java)).thenReturn(criteriaQuery) + whenever(criteriaQuery.from(any>())).thenReturn(root as Root) + whenever(criteriaQuery.select(any())).thenReturn(criteriaQuery) + whenever(entityManager.createQuery(criteriaQuery)).thenReturn(typedQuery) + whenever(typedQuery.singleResult).thenReturn(100L) + } + + @Test + fun `startOrResume creates a new RUNNING run`() { + val request = ReindexRequest(documentDefinitionName = "house", pageSize = 250) + + val run = service.startOrResume(request) + + assertThat(run.status).isEqualTo(ReindexRunStatus.RUNNING) + assertThat(run.pageSize).isEqualTo(250) + assertThat(run.scope).contains("house") + verify(repository).save(any()) + } + + @Test + fun `startOrResume re-arms an existing run when resumeRunId is set`() { + val runId = UUID.randomUUID() + val existing = OpenSearchReindexRun( + id = runId, + status = ReindexRunStatus.FAILED, + pageSize = 100, + lastId = UUID.randomUUID(), + error = "boom", + ) + whenever(repository.findById(runId)).thenReturn(Optional.of(existing)) + + val run = service.startOrResume(ReindexRequest(resumeRunId = runId)) + + assertThat(run.id).isEqualTo(runId) + assertThat(run.status).isEqualTo(ReindexRunStatus.RUNNING) + assertThat(run.error).isNull() + assertThat(run.finishedOn).isNull() + } + + @Test + fun `recordProgress updates cursor and counts`() { + val runId = UUID.randomUUID() + val run = OpenSearchReindexRun(id = runId, pageSize = 100) + whenever(repository.findById(runId)).thenReturn(Optional.of(run)) + val cursor = UUID.randomUUID() + + service.recordProgress(runId, cursor, processed = 42, skipped = 3) + + assertThat(run.lastId).isEqualTo(cursor) + assertThat(run.processedCount).isEqualTo(42) + assertThat(run.skippedCount).isEqualTo(3) + verify(repository).save(run) + } + + @Test + fun `complete fail and stop set the terminal status`() { + val runId = UUID.randomUUID() + val run = OpenSearchReindexRun(id = runId, pageSize = 100) + whenever(repository.findById(runId)).thenReturn(Optional.of(run)) + + service.complete(runId, 42L) + assertThat(run.status).isEqualTo(ReindexRunStatus.COMPLETED) + assertThat(run.finishedOn).isNotNull() + assertThat(run.totalCount).isEqualTo(42L) + + service.fail(runId, "kaboom") + assertThat(run.status).isEqualTo(ReindexRunStatus.FAILED) + assertThat(run.error).isEqualTo("kaboom") + + service.stop(runId) + assertThat(run.status).isEqualTo(ReindexRunStatus.STOPPED) + } + + @Test + fun `reconcileOrphanedRuns marks stale-heartbeat RUNNING rows as FAILED`() { + val orphan = OpenSearchReindexRun( + id = UUID.randomUUID(), + status = ReindexRunStatus.RUNNING, + pageSize = 100, + ) + whenever(repository.findAllByStatusAndHeartbeatOnBefore(eq(ReindexRunStatus.RUNNING), any())) + .thenReturn(listOf(orphan)) + + service.reconcileOrphanedRuns() + + assertThat(orphan.status).isEqualTo(ReindexRunStatus.FAILED) + assertThat(orphan.error).contains("Reconciled on startup") + val captor = argumentCaptor>() + verify(repository).saveAll(captor.capture()) + assertThat(captor.firstValue).containsExactly(orphan) + } + + @Test + fun `reconcileOrphanedRuns does nothing when no orphans exist`() { + whenever(repository.findAllByStatusAndHeartbeatOnBefore(eq(ReindexRunStatus.RUNNING), any())) + .thenReturn(emptyList()) + + service.reconcileOrphanedRuns() + + verify(repository, never()).saveAll(any>()) + } + + @Test + fun `toStatusMap returns a not-running placeholder when nothing matches`() { + whenever(repository.findFirstByOrderByStartedOnDesc()).thenReturn(null) + + val status = service.toStatusMap(null) + + assertThat(status["running"]).isEqualTo(false) + assertThat(status["runId"]).isNull() + } + + @Test + fun `toStatusMap reports running state and counts for a specific run`() { + val runId = UUID.randomUUID() + val run = OpenSearchReindexRun( + id = runId, + status = ReindexRunStatus.RUNNING, + pageSize = 100, + processedCount = 7, + skippedCount = 1, + ) + whenever(repository.findById(runId)).thenReturn(Optional.of(run)) + + val status = service.toStatusMap(runId) + + assertThat(status["runId"]).isEqualTo(runId) + assertThat(status["running"]).isEqualTo(true) + assertThat(status["status"]).isEqualTo(ReindexRunStatus.RUNNING) + assertThat(status["processedCount"]).isEqualTo(7L) + assertThat(status["skippedCount"]).isEqualTo(1L) + } +} diff --git a/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/service/ReindexProgressGateTest.kt b/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/service/ReindexProgressGateTest.kt new file mode 100644 index 0000000000..acaeec9786 --- /dev/null +++ b/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/service/ReindexProgressGateTest.kt @@ -0,0 +1,76 @@ +/* + * Copyright 2015-2026 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.ritense.document.opensearch.service + +import com.ritense.document.opensearch.OpenSearchProperties +import org.assertj.core.api.Assertions.assertThat +import org.junit.jupiter.api.Test +import org.mockito.kotlin.any +import org.mockito.kotlin.mock +import org.mockito.kotlin.never +import org.mockito.kotlin.times +import org.mockito.kotlin.verify +import org.mockito.kotlin.whenever + +class ReindexProgressGateTest { + + private val reindexRunService: OpenSearchReindexRunService = mock() + + @Test + fun `reports in-progress when a reindex run is running`() { + whenever(reindexRunService.isReindexRunning(any())).thenReturn(true) + val gate = ReindexProgressGate(reindexRunService, OpenSearchProperties()) + + assertThat(gate.isReindexInProgress()).isTrue() + } + + @Test + fun `reports not in-progress when no reindex run is running`() { + whenever(reindexRunService.isReindexRunning(any())).thenReturn(false) + val gate = ReindexProgressGate(reindexRunService, OpenSearchProperties()) + + assertThat(gate.isReindexInProgress()).isFalse() + } + + @Test + fun `never queries the run service when fallback is disabled`() { + val properties = OpenSearchProperties( + reindex = OpenSearchProperties.Reindex(fallbackToPostgresWhileRunning = false) + ) + val gate = ReindexProgressGate(reindexRunService, properties) + + assertThat(gate.isReindexInProgress()).isFalse() + verify(reindexRunService, never()).isReindexRunning(any()) + } + + @Test + fun `caches the result within the ttl window and refreshes after it`() { + whenever(reindexRunService.isReindexRunning(any())).thenReturn(true) + var now = 1_000L + val gate = ReindexProgressGate(reindexRunService, OpenSearchProperties(), clock = { now }) + + gate.isReindexInProgress() + gate.isReindexInProgress() + // Within the TTL: only one DB check. + verify(reindexRunService, times(1)).isReindexRunning(any()) + + now += ReindexProgressGate.CACHE_TTL_MS + gate.isReindexInProgress() + // TTL elapsed: a second check. + verify(reindexRunService, times(2)).isReindexRunning(any()) + } +} diff --git a/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/service/ReindexRequestTest.kt b/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/service/ReindexRequestTest.kt new file mode 100644 index 0000000000..3f3ef082e5 --- /dev/null +++ b/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/service/ReindexRequestTest.kt @@ -0,0 +1,46 @@ +/* + * Copyright 2015-2026 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.ritense.document.opensearch.service + +import org.assertj.core.api.Assertions.assertThat +import org.junit.jupiter.api.Test + +class ReindexRequestTest { + + @Test + fun `effectivePageSize clamps values above the maximum`() { + val request = ReindexRequest(pageSize = ReindexRequest.MAX_PAGE_SIZE + 5000) + + assertThat(request.effectivePageSize()).isEqualTo(ReindexRequest.MAX_PAGE_SIZE) + } + + @Test + fun `effectivePageSize clamps zero and negative values to one`() { + assertThat(ReindexRequest(pageSize = 0).effectivePageSize()).isEqualTo(1) + assertThat(ReindexRequest(pageSize = -10).effectivePageSize()).isEqualTo(1) + } + + @Test + fun `effectivePageSize keeps values within range`() { + assertThat(ReindexRequest(pageSize = 1234).effectivePageSize()).isEqualTo(1234) + } + + @Test + fun `default page size is used when not specified`() { + assertThat(ReindexRequest().effectivePageSize()).isEqualTo(ReindexRequest.DEFAULT_PAGE_SIZE) + } +} diff --git a/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/service/SearchEngineToggleTest.kt b/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/service/SearchEngineToggleTest.kt new file mode 100644 index 0000000000..028641bab9 --- /dev/null +++ b/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/service/SearchEngineToggleTest.kt @@ -0,0 +1,56 @@ +/* + * Copyright 2015-2024 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.ritense.document.opensearch.service + +import org.assertj.core.api.Assertions.assertThat +import org.junit.jupiter.api.Test + +class SearchEngineToggleTest { + + @Test + fun `default engine is OPENSEARCH`() { + val toggle = SearchEngineToggle() + + assertThat(toggle.get()).isEqualTo(SearchEngineToggle.Engine.OPENSEARCH) + } + + @Test + fun `can override default engine`() { + val toggle = SearchEngineToggle(default = SearchEngineToggle.Engine.POSTGRES) + + assertThat(toggle.get()).isEqualTo(SearchEngineToggle.Engine.POSTGRES) + } + + @Test + fun `set changes engine`() { + val toggle = SearchEngineToggle() + + toggle.set(SearchEngineToggle.Engine.POSTGRES) + + assertThat(toggle.get()).isEqualTo(SearchEngineToggle.Engine.POSTGRES) + } + + @Test + fun `can toggle back to OPENSEARCH`() { + val toggle = SearchEngineToggle() + toggle.set(SearchEngineToggle.Engine.POSTGRES) + + toggle.set(SearchEngineToggle.Engine.OPENSEARCH) + + assertThat(toggle.get()).isEqualTo(SearchEngineToggle.Engine.OPENSEARCH) + } +} diff --git a/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/web/DocumentOpenSearchReindexResourceTest.kt b/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/web/DocumentOpenSearchReindexResourceTest.kt new file mode 100644 index 0000000000..75838859f7 --- /dev/null +++ b/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/web/DocumentOpenSearchReindexResourceTest.kt @@ -0,0 +1,113 @@ +/* + * Copyright 2015-2026 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.ritense.document.opensearch.web + +import com.ritense.document.opensearch.service.DocumentOpenSearchReindexService +import com.ritense.document.opensearch.service.ReindexRequest +import org.assertj.core.api.Assertions.assertThat +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test +import org.mockito.kotlin.any +import org.mockito.kotlin.mock +import org.mockito.kotlin.whenever +import org.springframework.data.domain.PageImpl +import org.springframework.data.domain.PageRequest +import java.util.UUID + +class DocumentOpenSearchReindexResourceTest { + + private val reindexService: DocumentOpenSearchReindexService = mock() + private lateinit var resource: DocumentOpenSearchReindexResource + + @BeforeEach + fun setUp() { + resource = DocumentOpenSearchReindexResource(reindexService) + } + + @Test + fun `reindex returns 202 with the run id when started`() { + val runId = UUID.randomUUID() + whenever(reindexService.start(any())).thenReturn(runId) + + val response = resource.reindex(ReindexRequest(documentDefinitionName = "house")) + + assertThat(response.statusCode.value()).isEqualTo(202) + assertThat(response.body?.get("status")).isEqualTo("started") + assertThat(response.body?.get("runId")).isEqualTo(runId) + } + + @Test + fun `reindex starts with an empty request when no body is provided`() { + val runId = UUID.randomUUID() + whenever(reindexService.start(any())).thenReturn(runId) + + val response = resource.reindex(null) + + assertThat(response.statusCode.value()).isEqualTo(202) + assertThat(response.body?.get("runId")).isEqualTo(runId) + } + + @Test + fun `reindex returns 409 when a re-index is already running`() { + whenever(reindexService.start(any())).thenReturn(null) + + val response = resource.reindex(ReindexRequest()) + + assertThat(response.statusCode.value()).isEqualTo(409) + assertThat(response.body?.get("error")).isEqualTo("Re-index already in progress") + } + + @Test + fun `status returns the most recent run`() { + whenever(reindexService.status(null)).thenReturn(mapOf("running" to true)) + + val response = resource.status() + + assertThat(response.statusCode.value()).isEqualTo(200) + assertThat(response.body?.get("running")).isEqualTo(true) + } + + @Test + fun `statusById returns the requested run`() { + val runId = UUID.randomUUID() + whenever(reindexService.status(runId)).thenReturn(mapOf("runId" to runId)) + + val response = resource.statusById(runId) + + assertThat(response.statusCode.value()).isEqualTo(200) + assertThat(response.body?.get("runId")).isEqualTo(runId) + } + + @Test + fun `listRuns returns paginated runs`() { + val runId1 = UUID.randomUUID() + val runId2 = UUID.randomUUID() + val runs: List> = listOf( + mapOf("runId" to runId1, "status" to "COMPLETED"), + mapOf("runId" to runId2, "status" to "RUNNING") + ) + val page = PageImpl(runs, PageRequest.of(0, 20), 2) + whenever(reindexService.listRuns(PageRequest.of(0, 20))).thenReturn(page) + + val response = resource.listRuns(0, 20) + + assertThat(response.statusCode.value()).isEqualTo(200) + assertThat(response.body?.content).hasSize(2) + assertThat(response.body?.content?.get(0)?.get("runId")).isEqualTo(runId1) + assertThat(response.body?.totalElements).isEqualTo(2) + } +} diff --git a/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/web/SearchEngineResourceTest.kt b/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/web/SearchEngineResourceTest.kt new file mode 100644 index 0000000000..2aa2adc161 --- /dev/null +++ b/backend/case-opensearch/src/test/kotlin/com/ritense/document/opensearch/web/SearchEngineResourceTest.kt @@ -0,0 +1,118 @@ +/* + * Copyright 2015-2024 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.ritense.document.opensearch.web + +import com.ritense.adminsettings.service.FeatureToggleOverridesService +import com.ritense.adminsettings.web.rest.dto.FeatureToggleOverridesDto +import com.ritense.document.opensearch.OpenSearchProperties +import com.ritense.document.opensearch.service.DocumentOpenSearchIndexInitializer +import com.ritense.document.opensearch.service.SearchEngineToggle +import org.assertj.core.api.Assertions.assertThat +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test +import org.mockito.kotlin.any +import org.mockito.kotlin.eq +import org.mockito.kotlin.mock +import org.mockito.kotlin.never +import org.mockito.kotlin.verify +import org.mockito.kotlin.whenever +import org.springframework.http.HttpStatus + +class SearchEngineResourceTest { + + private lateinit var toggle: SearchEngineToggle + private lateinit var properties: OpenSearchProperties + private lateinit var featureToggleService: FeatureToggleOverridesService + private lateinit var indexInitializer: DocumentOpenSearchIndexInitializer + private lateinit var resource: SearchEngineResource + + @BeforeEach + fun setUp() { + toggle = SearchEngineToggle() + properties = OpenSearchProperties(enabled = true) + featureToggleService = mock() + indexInitializer = mock() + resource = SearchEngineResource(toggle, properties, featureToggleService, indexInitializer) + } + + @Test + fun `getActive returns available true and current engine when OpenSearch enabled`() { + toggle.set(SearchEngineToggle.Engine.OPENSEARCH) + + val response = resource.getActive() + + assertThat(response.statusCode).isEqualTo(HttpStatus.OK) + assertThat(response.body?.available).isTrue() + assertThat(response.body?.active).isEqualTo("OPENSEARCH") + } + + @Test + fun `getActive returns available false when OpenSearch disabled`() { + val disabledResource = SearchEngineResource( + toggle, + OpenSearchProperties(enabled = false), + featureToggleService, + indexInitializer + ) + + val response = disabledResource.getActive() + + assertThat(response.body?.available).isFalse() + } + + @Test + fun `setActive updates toggle and persists to feature toggles`() { + whenever(featureToggleService.updateToggle(any(), any())) + .thenReturn(FeatureToggleOverridesDto(mapOf("useOpenSearchForDocumentSearch" to false))) + + val response = resource.setActive(SearchEngineResource.UpdateSearchEngineDto("POSTGRES")) + + assertThat(response.statusCode).isEqualTo(HttpStatus.OK) + assertThat(response.body?.active).isEqualTo("POSTGRES") + assertThat(toggle.get()).isEqualTo(SearchEngineToggle.Engine.POSTGRES) + verify(featureToggleService).updateToggle(eq("useOpenSearchForDocumentSearch"), eq(false)) + verify(indexInitializer, never()).ensureIndex() + } + + @Test + fun `setActive to OPENSEARCH persists true`() { + toggle.set(SearchEngineToggle.Engine.POSTGRES) + whenever(featureToggleService.updateToggle(any(), any())) + .thenReturn(FeatureToggleOverridesDto(mapOf("useOpenSearchForDocumentSearch" to true))) + + val response = resource.setActive(SearchEngineResource.UpdateSearchEngineDto("OPENSEARCH")) + + assertThat(response.body?.active).isEqualTo("OPENSEARCH") + assertThat(toggle.get()).isEqualTo(SearchEngineToggle.Engine.OPENSEARCH) + verify(featureToggleService).updateToggle(eq("useOpenSearchForDocumentSearch"), eq(true)) + verify(indexInitializer).ensureIndex() + } + + @Test + fun `setActive returns bad request when OpenSearch disabled`() { + val disabledResource = SearchEngineResource( + toggle, + OpenSearchProperties(enabled = false), + featureToggleService, + indexInitializer + ) + + val response = disabledResource.setActive(SearchEngineResource.UpdateSearchEngineDto("OPENSEARCH")) + + assertThat(response.statusCode).isEqualTo(HttpStatus.BAD_REQUEST) + } +} diff --git a/backend/case-opensearch/src/test/resources/config/application-postgresql.yml b/backend/case-opensearch/src/test/resources/config/application-postgresql.yml new file mode 100644 index 0000000000..88f952877c --- /dev/null +++ b/backend/case-opensearch/src/test/resources/config/application-postgresql.yml @@ -0,0 +1,21 @@ +spring: + datasource: + driver-class-name: org.postgresql.Driver + url: jdbc:postgresql://localhost:3365/case-opensearch-test + username: valtimo + password: password + hikari: + auto-commit: false + jpa: + database-platform: org.hibernate.dialect.PostgreSQLDialect + database: postgresql + elasticsearch: + uris: http://localhost:39200 + +# The spring-data-opensearch starter binds its own client from the `opensearch.*` namespace +# (not spring.elasticsearch.*), defaulting to localhost:9200. The test OpenSearch is on 39200. +opensearch: + uris: http://localhost:39200 + +valtimo: + database: postgres diff --git a/backend/case-opensearch/src/test/resources/config/application.yml b/backend/case-opensearch/src/test/resources/config/application.yml new file mode 100644 index 0000000000..c8e281e47c --- /dev/null +++ b/backend/case-opensearch/src/test/resources/config/application.yml @@ -0,0 +1,41 @@ +spring: + datasource: + type: com.zaxxer.hikari.HikariDataSource + liquibase: + enabled: false + jpa: + show_sql: false + open-in-view: false + properties: + hibernate: + hbm2ddl.auto: none + format_sql: true + jdbc: + time_zone: UTC + connection: + provider_disables_autocommit: true + hibernate: + ddl-auto: none + +spring-actuator: + username: test + password: test + +valtimo: + versioning: + enabled: false + plugin: + encryption-secret: "abcdefghijklmnop" + opensearch: + enabled: true + reconcile: + # Disable the scheduled reconcile job in tests so it cannot interfere with assertions; the tests + # invoke DocumentOpenSearchReconcileService.reconcile() directly instead. + enabled: false + +operaton: + bpm: + history-level: audit + generic-properties: + properties: + enforceHistoryTimeToLive: false diff --git a/backend/case-opensearch/src/test/resources/config/case/house/1-0-0/case/definition/house.case-definition.json b/backend/case-opensearch/src/test/resources/config/case/house/1-0-0/case/definition/house.case-definition.json new file mode 100644 index 0000000000..994804cbd1 --- /dev/null +++ b/backend/case-opensearch/src/test/resources/config/case/house/1-0-0/case/definition/house.case-definition.json @@ -0,0 +1,7 @@ +{ + "key": "house", + "name": "House", + "versionTag": "1.0.0", + "canHaveAssignee": true, + "autoAssignTasks": true +} \ No newline at end of file diff --git a/backend/case-opensearch/src/test/resources/config/case/house/1-0-0/case/internal-status/house.internal-case-status.json b/backend/case-opensearch/src/test/resources/config/case/house/1-0-0/case/internal-status/house.internal-case-status.json new file mode 100644 index 0000000000..b69712429a --- /dev/null +++ b/backend/case-opensearch/src/test/resources/config/case/house/1-0-0/case/internal-status/house.internal-case-status.json @@ -0,0 +1,20 @@ +[ + { + "key": "suspended", + "title": "Suspended", + "visibleInCaseListByDefault": false, + "color": "GRAY" + }, + { + "key": "closed", + "title": "Closed", + "visibleInCaseListByDefault": false, + "color": "GRAY" + }, + { + "key": "started", + "title": "Started", + "visibleInCaseListByDefault": true, + "color": "GRAY" + } +] diff --git a/backend/case-opensearch/src/test/resources/config/case/house/1-0-0/case/list/house.case-list.json b/backend/case-opensearch/src/test/resources/config/case/house/1-0-0/case/list/house.case-list.json new file mode 100644 index 0000000000..fe51488c70 --- /dev/null +++ b/backend/case-opensearch/src/test/resources/config/case/house/1-0-0/case/list/house.case-list.json @@ -0,0 +1 @@ +[] diff --git a/backend/case-opensearch/src/test/resources/config/case/house/1-0-0/case/search-field/house.case-search-field.json b/backend/case-opensearch/src/test/resources/config/case/house/1-0-0/case/search-field/house.case-search-field.json new file mode 100644 index 0000000000..9f35ab434f --- /dev/null +++ b/backend/case-opensearch/src/test/resources/config/case/house/1-0-0/case/search-field/house.case-search-field.json @@ -0,0 +1,25 @@ +{ + "searchFields": [ + { + "key": "buildDate", + "path": "doc:buildDate", + "dataType": "date", + "fieldType": "single", + "matchType": "exact" + }, + { + "key": "buildDates", + "path": "doc:buildDate", + "dataType": "date", + "fieldType": "range", + "matchType": "exact" + }, + { + "key": "street", + "path": "doc:street", + "dataType": "text", + "fieldType": "single", + "matchType": "like" + } + ] +} diff --git a/backend/case-opensearch/src/test/resources/config/case/house/1-0-0/document/definition/house.schema.document-definition.json b/backend/case-opensearch/src/test/resources/config/case/house/1-0-0/document/definition/house.schema.document-definition.json new file mode 100644 index 0000000000..965fb0c585 --- /dev/null +++ b/backend/case-opensearch/src/test/resources/config/case/house/1-0-0/document/definition/house.schema.document-definition.json @@ -0,0 +1,33 @@ +{ + "$id": "house.schema", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "House", + "type": "object", + "properties": { + "street": { + "type": "string", + "description": "The street name.", + "maxLength": 100 + }, + "housenumber": { + "description": "house number must be equal to or greater than zero.", + "type": "integer", + "minimum": 0 + }, + "buildDate": { + "type": "string", + "description": "The house's build date.", + "maxLength": 100 + }, + "userInfo": { + "type": "string", + "description": "Additional information on the user", + "maxLength": 100 + }, + "loan-approved": { + "type": "boolean", + "description": "Was the loan for the house approved" + } + }, + "additionalProperties": false +} diff --git a/backend/case-opensearch/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker b/backend/case-opensearch/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker new file mode 100644 index 0000000000..1f0955d450 --- /dev/null +++ b/backend/case-opensearch/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker @@ -0,0 +1 @@ +mock-maker-inline diff --git a/backend/case/src/main/java/com/ritense/document/domain/impl/JsonSchemaDocument.java b/backend/case/src/main/java/com/ritense/document/domain/impl/JsonSchemaDocument.java index f8d5060d2c..d2f5bf177f 100644 --- a/backend/case/src/main/java/com/ritense/document/domain/impl/JsonSchemaDocument.java +++ b/backend/case/src/main/java/com/ritense/document/domain/impl/JsonSchemaDocument.java @@ -1,5 +1,5 @@ /* - * Copyright 2015-2024 Ritense BV, the Netherlands. + * Copyright 2015-2026 Ritense BV, the Netherlands. * * Licensed under EUPL, Version 1.2 (the "License"); * you may not use this file except in compliance with the License. @@ -54,6 +54,8 @@ import jakarta.persistence.JoinTable; import jakarta.persistence.ManyToMany; import jakarta.persistence.ManyToOne; +import jakarta.persistence.PrePersist; +import jakarta.persistence.PreUpdate; import jakarta.persistence.Table; import jakarta.persistence.Version; import java.time.LocalDateTime; @@ -108,6 +110,9 @@ public class JsonSchemaDocument extends AbstractAggregateRoot modifiedOn() { return Optional.ofNullable(modifiedOn); } + @JsonIgnore + public LocalDateTime changedOn() { + return changedOn; + } + + @PrePersist + @PreUpdate + void updateChangedOn() { + this.changedOn = LocalDateTime.now(); + } + @Override public JsonDocumentContent content() { return content; diff --git a/backend/case/src/main/java/com/ritense/document/domain/search/AdvancedSearchRequest.java b/backend/case/src/main/java/com/ritense/document/domain/search/AdvancedSearchRequest.java index 356115c1d9..3398c8939f 100644 --- a/backend/case/src/main/java/com/ritense/document/domain/search/AdvancedSearchRequest.java +++ b/backend/case/src/main/java/com/ritense/document/domain/search/AdvancedSearchRequest.java @@ -28,6 +28,7 @@ public class AdvancedSearchRequest { private List otherFilters = List.of(); private Set statusFilter = new HashSet<>(); private Set caseTagsFilter = new HashSet<>(); + private String globalSearchFilter; public AdvancedSearchRequest() { // Jackson needs the empty constructor @@ -94,6 +95,19 @@ public void setCaseTagsFilter(Set caseTagsFilter) { this.caseTagsFilter = caseTagsFilter != null ? caseTagsFilter : new HashSet<>(); } + public String getGlobalSearchFilter() { + return globalSearchFilter; + } + + public void setGlobalSearchFilter(String globalSearchFilter) { + this.globalSearchFilter = globalSearchFilter; + } + + public AdvancedSearchRequest globalSearchFilter(String globalSearchFilter) { + setGlobalSearchFilter(globalSearchFilter); + return this; + } + public static class OtherFilter { private String path; diff --git a/backend/case/src/main/java/com/ritense/document/domain/search/SearchRequestMapper.java b/backend/case/src/main/java/com/ritense/document/domain/search/SearchRequestMapper.java index ac833a2780..0068db3892 100644 --- a/backend/case/src/main/java/com/ritense/document/domain/search/SearchRequestMapper.java +++ b/backend/case/src/main/java/com/ritense/document/domain/search/SearchRequestMapper.java @@ -73,6 +73,7 @@ public static AdvancedSearchRequest toAdvancedSearchRequest(SearchWithConfigRequ advancedSearchRequest.setOtherFilters(otherFilters); advancedSearchRequest.setStatusFilter(searchRequest.getStatusFilter()); advancedSearchRequest.setCaseTagsFilter(searchRequest.getCaseTagsFilter()); + advancedSearchRequest.setGlobalSearchFilter(searchRequest.getGlobalSearchFilter()); return advancedSearchRequest; } diff --git a/backend/case/src/main/java/com/ritense/document/domain/search/SearchWithConfigRequest.java b/backend/case/src/main/java/com/ritense/document/domain/search/SearchWithConfigRequest.java index 323e8d33c9..abc3598e72 100644 --- a/backend/case/src/main/java/com/ritense/document/domain/search/SearchWithConfigRequest.java +++ b/backend/case/src/main/java/com/ritense/document/domain/search/SearchWithConfigRequest.java @@ -29,6 +29,7 @@ public class SearchWithConfigRequest { private List otherFilters = List.of(); private Set statusFilter = Set.of(); private Set caseTagsFilter = Set.of(); + private String globalSearchFilter; public SearchWithConfigRequest() { } @@ -85,6 +86,14 @@ public void setCaseTagsFilter(Set caseTagsFilter) { this.caseTagsFilter = caseTagsFilter; } + public String getGlobalSearchFilter() { + return globalSearchFilter; + } + + public void setGlobalSearchFilter(String globalSearchFilter) { + this.globalSearchFilter = globalSearchFilter; + } + public static class SearchWithConfigFilter { private String key; diff --git a/backend/case/src/main/java/com/ritense/document/service/DocumentSearchService.java b/backend/case/src/main/java/com/ritense/document/service/DocumentSearchService.java index e110738ad5..d91ba2509a 100644 --- a/backend/case/src/main/java/com/ritense/document/service/DocumentSearchService.java +++ b/backend/case/src/main/java/com/ritense/document/service/DocumentSearchService.java @@ -55,4 +55,14 @@ Long count( AdvancedSearchRequest advancedSearchRequest ); + @SuppressWarnings({"squid:S1452", "java:S1452"}) + default Page searchForExport( + String documentDefinitionName, + BlueprintType blueprintType, + SearchWithConfigRequest searchWithConfigRequest, + Pageable pageable + ) { + return search(documentDefinitionName, blueprintType, searchWithConfigRequest, pageable); + } + } diff --git a/backend/case/src/main/java/com/ritense/document/service/impl/JsonSchemaDocumentSearchService.java b/backend/case/src/main/java/com/ritense/document/service/impl/JsonSchemaDocumentSearchService.java index d35b432049..4b74466ab1 100644 --- a/backend/case/src/main/java/com/ritense/document/service/impl/JsonSchemaDocumentSearchService.java +++ b/backend/case/src/main/java/com/ritense/document/service/impl/JsonSchemaDocumentSearchService.java @@ -72,8 +72,11 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import java.util.stream.Collectors; import java.util.stream.Stream; +import com.ritense.document.domain.impl.searchfield.SearchFieldMatchType; import org.apache.commons.lang3.NotImplementedException; import org.apache.commons.lang3.StringUtils; import org.springframework.data.domain.Page; @@ -394,6 +397,97 @@ private void buildQueryWhere( predicates.add(getCaseTagsFilterPredicate(cb, documentRoot, searchRequest.getCaseTagsFilter())); } + if (searchRequest.getGlobalSearchFilter() != null && !searchRequest.getGlobalSearchFilter().isBlank()) { + var searchFields = !StringUtils.isEmpty(documentDefinitionName) + ? searchFieldService.getSearchFields(documentDefinitionName) + : List.of(); + + if (searchFields.isEmpty()) { + predicates.add(cb.disjunction()); + } else { + var fieldMap = searchFields.stream() + .collect(Collectors.toMap(f -> removePrefixes(f.getPath()), f -> f, (a, b) -> a)); + + var docFields = searchFields.stream() + .filter(f -> f.getPath() != null && f.getPath().startsWith(DOC_PREFIX)) + .toList(); + + var caseTextFields = searchFields.stream() + .filter(f -> f.getPath() != null && f.getPath().startsWith(CASE_PREFIX)) + .filter(f -> f.getDataType() == SearchFieldDataType.TEXT) + .toList(); + + var parsedTerms = parseGlobalSearch(searchRequest.getGlobalSearchFilter()); + + List qualifiedPredicates = new ArrayList<>(); + List unqualifiedPredicates = new ArrayList<>(); + + for (ParsedTerm term : parsedTerms) { + if (term.field() != null) { + var fieldPath = removePrefixes(term.field()); + var field = fieldMap.get(fieldPath); + if (field == null) { + throw new IllegalArgumentException("Unknown search field: " + term.field()); + } + + boolean isDocField = field.getPath().startsWith(DOC_PREFIX); + + if (isDocField) { + var jsonPath = "$." + fieldPath; + Expression expr = queryDialectHelper.getJsonValueExpression( + cb, documentRoot.get(CONTENT).get(CONTENT), jsonPath, String.class + ); + var likePattern = buildLikePattern(term.value(), term.quoted(), field.getMatchType()); + qualifiedPredicates.add(cb.like(cb.lower(expr), likePattern.toLowerCase())); + } else { + if (field.getDataType() == SearchFieldDataType.DATE || + field.getDataType() == SearchFieldDataType.DATETIME) { + var date = LocalDate.parse(term.value()); + var startOfDay = date.atStartOfDay(); + var endOfDay = date.plusDays(1).atStartOfDay(); + qualifiedPredicates.add(cb.and( + cb.greaterThanOrEqualTo(documentRoot.get(fieldPath), startOfDay), + cb.lessThan(documentRoot.get(fieldPath), endOfDay) + )); + } else { + var likePattern = buildLikePattern(term.value(), term.quoted(), field.getMatchType()); + qualifiedPredicates.add(cb.like( + cb.lower(documentRoot.get(fieldPath).as(String.class)), + likePattern.toLowerCase() + )); + } + } + } else { + var likePattern = "%" + queryDialectHelper.escapeLikePattern(term.value()).toLowerCase() + "%"; + List termPredicates = new ArrayList<>(); + + for (var f : docFields) { + var jsonPath = "$." + f.getPath().substring(DOC_PREFIX.length()); + Expression expr = queryDialectHelper.getJsonValueExpression( + cb, documentRoot.get(CONTENT).get(CONTENT), jsonPath, String.class + ); + termPredicates.add(cb.like(cb.lower(expr), likePattern)); + } + + for (var f : caseTextFields) { + var columnName = f.getPath().substring(CASE_PREFIX.length()); + termPredicates.add(cb.like( + cb.lower(documentRoot.get(columnName).as(String.class)), + likePattern + )); + } + + if (!termPredicates.isEmpty()) { + unqualifiedPredicates.add(cb.or(termPredicates.toArray(Predicate[]::new))); + } + } + } + + qualifiedPredicates.forEach(predicates::add); + unqualifiedPredicates.forEach(predicates::add); + } + } + query.where(predicates.toArray(Predicate[]::new)); } @@ -812,4 +906,75 @@ void apply( Root documentRoot ); } + + private record ParsedTerm(String field, String value, boolean quoted) {} + + private List parseGlobalSearch(String query) { + List terms = new ArrayList<>(); + Pattern fieldPattern = Pattern.compile("(\\w+(?:\\.\\w+)*):(\"([^\"]+)\"|(\\S+))"); + Matcher matcher = fieldPattern.matcher(query); + + int lastEnd = 0; + while (matcher.find()) { + String before = query.substring(lastEnd, matcher.start()).trim(); + if (!before.isEmpty()) { + terms.addAll(parseUnqualifiedTerms(before)); + } + + String fieldName = matcher.group(1); + boolean quoted = matcher.group(3) != null && !matcher.group(3).isEmpty(); + String value = quoted ? matcher.group(3) : matcher.group(4); + + terms.add(new ParsedTerm(fieldName, value, quoted)); + lastEnd = matcher.end(); + } + + String after = query.substring(lastEnd).trim(); + if (!after.isEmpty()) { + terms.addAll(parseUnqualifiedTerms(after)); + } + + return terms; + } + + private List parseUnqualifiedTerms(String text) { + List terms = new ArrayList<>(); + Pattern quotedPattern = Pattern.compile("\"([^\"]+)\""); + Matcher matcher = quotedPattern.matcher(text); + + int lastEnd = 0; + while (matcher.find()) { + String before = text.substring(lastEnd, matcher.start()).trim(); + if (!before.isEmpty()) { + Arrays.stream(before.split("\\s+")) + .filter(s -> !s.isEmpty()) + .forEach(s -> terms.add(new ParsedTerm(null, s, false))); + } + terms.add(new ParsedTerm(null, matcher.group(1), true)); + lastEnd = matcher.end(); + } + + String after = text.substring(lastEnd).trim(); + if (!after.isEmpty()) { + Arrays.stream(after.split("\\s+")) + .filter(s -> !s.isEmpty()) + .forEach(s -> terms.add(new ParsedTerm(null, s, false))); + } + + return terms; + } + + private String removePrefixes(String path) { + if (path == null) return null; + if (path.startsWith(DOC_PREFIX)) return path.substring(DOC_PREFIX.length()); + if (path.startsWith(CASE_PREFIX)) return path.substring(CASE_PREFIX.length()); + return path; + } + + private String buildLikePattern(String value, boolean quoted, SearchFieldMatchType matchType) { + if (quoted || matchType != SearchFieldMatchType.LIKE) { + return value; + } + return "%" + queryDialectHelper.escapeLikePattern(value) + "%"; + } } diff --git a/backend/case/src/main/java/com/ritense/document/service/impl/JsonSchemaDocumentService.java b/backend/case/src/main/java/com/ritense/document/service/impl/JsonSchemaDocumentService.java index 4a09e51db4..35fc5e88ba 100644 --- a/backend/case/src/main/java/com/ritense/document/service/impl/JsonSchemaDocumentService.java +++ b/backend/case/src/main/java/com/ritense/document/service/impl/JsonSchemaDocumentService.java @@ -603,11 +603,16 @@ public void removeDocuments( }); documentRepository.saveAll(documents); documentRepository.deleteAll(documents); - documents.forEach(document -> outboxService.send(() -> - new DocumentDeleted( - document.id().toString() - ) - )); + documents.forEach(document -> { + // Per-document Spring event so bulk deletes are handled identically to single deletes + // (durable pending index deletion + best-effort live delete). Fires inside this transaction. + applicationEventPublisher.publishEvent(new DocumentDeletedEvent(document.id().getId())); + outboxService.send(() -> + new DocumentDeleted( + document.id().toString() + ) + ); + }); documentSequenceGeneratorService.deleteSequenceRecordBy(documentDefinitionName); } } diff --git a/backend/case/src/main/kotlin/com/ritense/case/configuration/CaseAutoConfiguration.kt b/backend/case/src/main/kotlin/com/ritense/case/configuration/CaseAutoConfiguration.kt index a2ee341061..c60e825d5b 100644 --- a/backend/case/src/main/kotlin/com/ritense/case/configuration/CaseAutoConfiguration.kt +++ b/backend/case/src/main/kotlin/com/ritense/case/configuration/CaseAutoConfiguration.kt @@ -70,7 +70,6 @@ import com.ritense.case_.service.ActiveCaseDefinitionService import com.ritense.document.service.DocumentDefinitionService import com.ritense.document.service.DocumentSearchService import com.ritense.document.service.DocumentService -import com.ritense.document.service.impl.JsonSchemaDocumentSearchService import com.ritense.exporter.ExportService import com.ritense.importer.ImportService import com.ritense.importer.ValtimoImportService @@ -439,7 +438,7 @@ class CaseAutoConfiguration { @ConditionalOnMissingBean(CaseExporter::class) fun caseExporter( caseDefinitionListColumnRepository: CaseDefinitionListColumnRepository, - documentSearchService: JsonSchemaDocumentSearchService, + documentSearchService: DocumentSearchService, outboxService: OutboxService, mapper: ObjectMapper, caseListRowMapper: CaseListRowMapper diff --git a/backend/case/src/main/kotlin/com/ritense/case/service/CaseExporter.kt b/backend/case/src/main/kotlin/com/ritense/case/service/CaseExporter.kt index b55b922bd8..731e89b5f0 100644 --- a/backend/case/src/main/kotlin/com/ritense/case/service/CaseExporter.kt +++ b/backend/case/src/main/kotlin/com/ritense/case/service/CaseExporter.kt @@ -28,7 +28,7 @@ import com.ritense.valtimo.contract.blueprint.BlueprintType import com.ritense.document.domain.impl.JsonSchemaDocument import com.ritense.document.domain.search.SearchWithConfigRequest import com.ritense.document.event.DocumentsExported -import com.ritense.document.service.impl.JsonSchemaDocumentSearchService +import com.ritense.document.service.DocumentSearchService import com.ritense.outbox.OutboxService import com.ritense.valtimo.contract.utils.SecurityUtils import io.github.oshai.kotlinlogging.KotlinLogging @@ -49,7 +49,7 @@ import kotlin.text.Charsets.UTF_8 @Transactional class CaseExporter( private val caseDefinitionListColumnRepository: CaseDefinitionListColumnRepository, - private val documentSearchService: JsonSchemaDocumentSearchService, + private val documentSearchService: DocumentSearchService, private val outboxService: OutboxService, private val mapper: ObjectMapper, private val caseListRowMapper: CaseListRowMapper diff --git a/backend/case/src/test/java/com/ritense/document/service/impl/JsonSchemaDocumentServiceTest.java b/backend/case/src/test/java/com/ritense/document/service/impl/JsonSchemaDocumentServiceTest.java index 77c62e4c5a..a5d262dff5 100644 --- a/backend/case/src/test/java/com/ritense/document/service/impl/JsonSchemaDocumentServiceTest.java +++ b/backend/case/src/test/java/com/ritense/document/service/impl/JsonSchemaDocumentServiceTest.java @@ -33,10 +33,10 @@ import com.ritense.document.domain.impl.JsonDocumentContent; import com.ritense.document.domain.impl.JsonSchemaDocument; import com.ritense.document.domain.impl.JsonSchemaDocumentDefinition; +import com.ritense.document.domain.impl.JsonSchemaDocumentId; import com.ritense.document.domain.impl.request.NewDocumentRequest; import com.ritense.document.event.DocumentAssigneeChangedEvent; import com.ritense.document.event.DocumentUnassignedEvent; -import com.ritense.document.domain.impl.JsonSchemaDocumentId; import com.ritense.document.repository.impl.JsonSchemaDocumentRepository; import com.ritense.document.service.CaseTagService; import com.ritense.document.service.InternalCaseStatusService; @@ -47,6 +47,7 @@ import com.ritense.valtimo.contract.authentication.TeamManagementService; import com.ritense.valtimo.contract.authentication.UserManagementService; import com.ritense.valtimo.contract.case_.CaseDefinitionId; +import com.ritense.valtimo.contract.event.DocumentDeletedEvent; import com.ritense.valtimo.contract.json.MapperSingleton; import com.ritense.valtimo.contract.resource.Resource; import jakarta.persistence.EntityManager; @@ -218,6 +219,10 @@ void shouldRemoveDocuments() { verify(documentRepository, times(1)).saveAll(eq(jsonSchemaDocuments.toList())); verify(documentRepository, times(1)).deleteAll(eq(jsonSchemaDocuments.toList())); verify(documentSequenceGeneratorService, times(1)).deleteSequenceRecordBy(eq(documentDefinitionName)); + // A per-document Spring event is published so bulk deletes reach OpenSearch (pending index deletion) like single deletes. + var captor = ArgumentCaptor.forClass(DocumentDeletedEvent.class); + verify(applicationEventPublisher, times(1)).publishEvent(captor.capture()); + assertEquals(jsonSchemaDocument.id().getId(), captor.getValue().getCaseDocumentId()); } @Test diff --git a/backend/case/src/test/kotlin/com/ritense/case/service/CaseExporterTest.kt b/backend/case/src/test/kotlin/com/ritense/case/service/CaseExporterTest.kt index 9e2eb009e5..680d8924ad 100644 --- a/backend/case/src/test/kotlin/com/ritense/case/service/CaseExporterTest.kt +++ b/backend/case/src/test/kotlin/com/ritense/case/service/CaseExporterTest.kt @@ -30,7 +30,7 @@ import com.ritense.document.domain.impl.JsonSchemaDocumentDefinition import com.ritense.document.domain.impl.JsonSchemaDocumentDefinitionId import com.ritense.document.domain.impl.JsonSchemaDocumentId import com.ritense.document.domain.search.SearchWithConfigRequest -import com.ritense.document.service.impl.JsonSchemaDocumentSearchService +import com.ritense.document.service.DocumentSearchService import com.ritense.outbox.OutboxService import com.ritense.search.domain.DisplayType import com.ritense.search.domain.EmptyDisplayTypeParameter @@ -58,7 +58,7 @@ import kotlin.text.Charsets.UTF_8 class CaseExporterTest : BaseTest() { private lateinit var caseDefinitionListColumnRepository: CaseDefinitionListColumnRepository - private lateinit var documentSearchService: JsonSchemaDocumentSearchService + private lateinit var documentSearchService: DocumentSearchService private lateinit var outboxService: OutboxService private lateinit var mapper: ObjectMapper private lateinit var caseListRowMapper: CaseListRowMapper diff --git a/backend/contract/src/main/java/com/ritense/valtimo/contract/database/MysqlQueryDialectHelper.java b/backend/contract/src/main/java/com/ritense/valtimo/contract/database/MysqlQueryDialectHelper.java index c985447c04..2b0885a9bb 100644 --- a/backend/contract/src/main/java/com/ritense/valtimo/contract/database/MysqlQueryDialectHelper.java +++ b/backend/contract/src/main/java/com/ritense/valtimo/contract/database/MysqlQueryDialectHelper.java @@ -51,7 +51,7 @@ public Expression getJsonValueExpression(CriteriaBuilder cb, Path column, @Override public Predicate getJsonValueExistsExpression(CriteriaBuilder cb, Path column, String value) { Expression searchColumn = column; - Expression searchValue = cb.literal("%" + value.trim() + "%"); + Expression searchValue = cb.literal("%" + escapeLikePattern(value.trim()) + "%"); if (column.getJavaType() == String.class || column.getJavaType() == Object.class) { searchColumn = cb.function(LOWER_CASE_FUNCTION, String.class, searchColumn); searchValue = cb.function(LOWER_CASE_FUNCTION, String.class, searchValue); @@ -72,7 +72,7 @@ public Predicate getJsonValueExistsInPathExpression(CriteriaBuilder cb, Path col String value) { Expression searchColumn = column; Expression searchPath = cb.literal(path); - Expression searchValue = cb.literal("%" + value.trim() + "%"); + Expression searchValue = cb.literal("%" + escapeLikePattern(value.trim()) + "%"); if (column.getJavaType() == String.class || column.getJavaType() == Object.class) { searchColumn = cb.function(LOWER_CASE_FUNCTION, String.class, searchColumn.as(String.class)); searchPath = cb.function(LOWER_CASE_FUNCTION, String.class, searchPath); diff --git a/backend/contract/src/main/java/com/ritense/valtimo/contract/database/PostgresQueryDialectHelper.java b/backend/contract/src/main/java/com/ritense/valtimo/contract/database/PostgresQueryDialectHelper.java index 888dd0e5b3..3fdbc442de 100644 --- a/backend/contract/src/main/java/com/ritense/valtimo/contract/database/PostgresQueryDialectHelper.java +++ b/backend/contract/src/main/java/com/ritense/valtimo/contract/database/PostgresQueryDialectHelper.java @@ -67,7 +67,7 @@ public Predicate getJsonValueExistsExpression(CriteriaBuilder cb, Path column, S cb.function( "jsonpath", String.class, - cb.literal("$.** ? (@ like_regex \"" + value + "\")") + cb.literal("$.** ? (@ like_regex \"" + escapeJsonPathRegex(value) + "\" flag \"i\")") ) ) ); @@ -81,7 +81,7 @@ public Predicate getJsonValueExistsInPathExpression(CriteriaBuilder cb, Path col String.class, getValueForPathText(cb, column, path) ), - "%" + value.toLowerCase() + "%" + "%" + escapeLikePattern(value.toLowerCase()) + "%" ); } @@ -139,4 +139,8 @@ private List splitPath(String path) { private Expression toJsonb(CriteriaBuilder cb, Path column) { return cb.function("to_jsonb", Object.class, column); } + + private String escapeJsonPathRegex(String value) { + return value.replace("\\", "\\\\").replace("\"", "\\\""); + } } diff --git a/backend/contract/src/main/java/com/ritense/valtimo/contract/database/QueryDialectHelper.java b/backend/contract/src/main/java/com/ritense/valtimo/contract/database/QueryDialectHelper.java index 10c48af06b..701f432602 100644 --- a/backend/contract/src/main/java/com/ritense/valtimo/contract/database/QueryDialectHelper.java +++ b/backend/contract/src/main/java/com/ritense/valtimo/contract/database/QueryDialectHelper.java @@ -35,4 +35,10 @@ public interface QueryDialectHelper { Expression uuidToString(CriteriaBuilder cb, Path column); Expression stringToUuid(CriteriaBuilder cb, Expression expression); + + default String escapeLikePattern(String value) { + return value.replace("\\", "\\\\") + .replace("%", "\\%") + .replace("_", "\\_"); + } } diff --git a/backend/contract/src/test/java/com/ritense/valtimo/contract/database/MysqlQueryDialectHelperTest.java b/backend/contract/src/test/java/com/ritense/valtimo/contract/database/MysqlQueryDialectHelperTest.java new file mode 100644 index 0000000000..221ddd9d55 --- /dev/null +++ b/backend/contract/src/test/java/com/ritense/valtimo/contract/database/MysqlQueryDialectHelperTest.java @@ -0,0 +1,62 @@ +/* + * Copyright 2015-2026 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.ritense.valtimo.contract.database; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +class MysqlQueryDialectHelperTest { + + private MysqlQueryDialectHelper helper; + + @BeforeEach + void setUp() { + helper = new MysqlQueryDialectHelper(); + } + + @Test + void escapeLikePatternShouldEscapePercent() { + String result = helper.escapeLikePattern("100%"); + assertEquals("100\\%", result); + } + + @Test + void escapeLikePatternShouldEscapeUnderscore() { + String result = helper.escapeLikePattern("test_value"); + assertEquals("test\\_value", result); + } + + @Test + void escapeLikePatternShouldEscapeBackslash() { + String result = helper.escapeLikePattern("path\\to\\file"); + assertEquals("path\\\\to\\\\file", result); + } + + @Test + void escapeLikePatternShouldEscapeAllSpecialChars() { + String result = helper.escapeLikePattern("100%_test\\"); + assertEquals("100\\%\\_test\\\\", result); + } + + @Test + void escapeLikePatternShouldHandleNormalInput() { + String result = helper.escapeLikePattern("normal search"); + assertEquals("normal search", result); + } +} diff --git a/backend/contract/src/test/java/com/ritense/valtimo/contract/database/PostgresQueryDialectHelperTest.java b/backend/contract/src/test/java/com/ritense/valtimo/contract/database/PostgresQueryDialectHelperTest.java new file mode 100644 index 0000000000..baf8359812 --- /dev/null +++ b/backend/contract/src/test/java/com/ritense/valtimo/contract/database/PostgresQueryDialectHelperTest.java @@ -0,0 +1,90 @@ +/* + * Copyright 2015-2026 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.ritense.valtimo.contract.database; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.lang.reflect.Method; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +class PostgresQueryDialectHelperTest { + + private PostgresQueryDialectHelper helper; + private Method escapeJsonPathRegexMethod; + + @BeforeEach + void setUp() throws Exception { + helper = new PostgresQueryDialectHelper(); + escapeJsonPathRegexMethod = PostgresQueryDialectHelper.class.getDeclaredMethod("escapeJsonPathRegex", String.class); + escapeJsonPathRegexMethod.setAccessible(true); + } + + @Test + void escapeJsonPathRegexShouldEscapeBackslash() throws Exception { + String result = (String) escapeJsonPathRegexMethod.invoke(helper, "test\\value"); + assertEquals("test\\\\value", result); + } + + @Test + void escapeJsonPathRegexShouldEscapeDoubleQuote() throws Exception { + String result = (String) escapeJsonPathRegexMethod.invoke(helper, "test\"value"); + assertEquals("test\\\"value", result); + } + + @Test + void escapeJsonPathRegexShouldEscapeBothBackslashAndQuote() throws Exception { + String result = (String) escapeJsonPathRegexMethod.invoke(helper, "test\\\"injection"); + assertEquals("test\\\\\\\"injection", result); + } + + @Test + void escapeJsonPathRegexShouldHandleNormalInput() throws Exception { + String result = (String) escapeJsonPathRegexMethod.invoke(helper, "normal search term"); + assertEquals("normal search term", result); + } + + @Test + void escapeLikePatternShouldEscapePercent() { + String result = helper.escapeLikePattern("100%"); + assertEquals("100\\%", result); + } + + @Test + void escapeLikePatternShouldEscapeUnderscore() { + String result = helper.escapeLikePattern("test_value"); + assertEquals("test\\_value", result); + } + + @Test + void escapeLikePatternShouldEscapeBackslash() { + String result = helper.escapeLikePattern("path\\to\\file"); + assertEquals("path\\\\to\\\\file", result); + } + + @Test + void escapeLikePatternShouldEscapeAllSpecialChars() { + String result = helper.escapeLikePattern("100%_test\\"); + assertEquals("100\\%\\_test\\\\", result); + } + + @Test + void escapeLikePatternShouldHandleNormalInput() { + String result = helper.escapeLikePattern("normal search"); + assertEquals("normal search", result); + } +} diff --git a/backend/core/src/main/resources/config/liquibase/13-22-0/20260330-form-flow-blueprint-support.xml b/backend/core/src/main/resources/config/liquibase/13-22-0/20260330-form-flow-blueprint-support.xml new file mode 100644 index 0000000000..9813e43841 --- /dev/null +++ b/backend/core/src/main/resources/config/liquibase/13-22-0/20260330-form-flow-blueprint-support.xml @@ -0,0 +1,145 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/backend/core/src/main/resources/config/liquibase/13-38-0/13-38-0-master.xml b/backend/core/src/main/resources/config/liquibase/13-38-0/13-38-0-master.xml new file mode 100644 index 0000000000..faf435b689 --- /dev/null +++ b/backend/core/src/main/resources/config/liquibase/13-38-0/13-38-0-master.xml @@ -0,0 +1,32 @@ + + + + + + + + + + + + + + + diff --git a/backend/core/src/main/resources/config/liquibase/13-38-0/20260630-create-reindex-run-table.xml b/backend/core/src/main/resources/config/liquibase/13-38-0/20260630-create-reindex-run-table.xml new file mode 100644 index 0000000000..8da40cf174 --- /dev/null +++ b/backend/core/src/main/resources/config/liquibase/13-38-0/20260630-create-reindex-run-table.xml @@ -0,0 +1,58 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/backend/core/src/main/resources/config/liquibase/13-38-0/20260701-add-changed-on-to-json-schema-document.xml b/backend/core/src/main/resources/config/liquibase/13-38-0/20260701-add-changed-on-to-json-schema-document.xml new file mode 100644 index 0000000000..0598b3baae --- /dev/null +++ b/backend/core/src/main/resources/config/liquibase/13-38-0/20260701-add-changed-on-to-json-schema-document.xml @@ -0,0 +1,42 @@ + + + + + + + + + + + + + + + + changed_on IS NULL + + + + + + + + diff --git a/backend/core/src/main/resources/config/liquibase/13-38-0/20260701-create-pending-index-deletion-table.xml b/backend/core/src/main/resources/config/liquibase/13-38-0/20260701-create-pending-index-deletion-table.xml new file mode 100644 index 0000000000..0c58c5cbdf --- /dev/null +++ b/backend/core/src/main/resources/config/liquibase/13-38-0/20260701-create-pending-index-deletion-table.xml @@ -0,0 +1,38 @@ + + + + + + + + + + + + + + + + + + + + diff --git a/backend/core/src/main/resources/config/liquibase/13-38-0/20260701-create-reconcile-state-table.xml b/backend/core/src/main/resources/config/liquibase/13-38-0/20260701-create-reconcile-state-table.xml new file mode 100644 index 0000000000..53413ff39c --- /dev/null +++ b/backend/core/src/main/resources/config/liquibase/13-38-0/20260701-create-reconcile-state-table.xml @@ -0,0 +1,35 @@ + + + + + + + + + + + + + + + + + diff --git a/backend/core/src/main/resources/config/liquibase/13-38-0/20260710-add-reindex-run-pruned-count.xml b/backend/core/src/main/resources/config/liquibase/13-38-0/20260710-add-reindex-run-pruned-count.xml new file mode 100644 index 0000000000..fc0554c233 --- /dev/null +++ b/backend/core/src/main/resources/config/liquibase/13-38-0/20260710-add-reindex-run-pruned-count.xml @@ -0,0 +1,32 @@ + + + + + + + + + + + + + + diff --git a/backend/core/src/main/resources/config/liquibase/13-38-0/20260713-add-reindex-run-total-count.xml b/backend/core/src/main/resources/config/liquibase/13-38-0/20260713-add-reindex-run-total-count.xml new file mode 100644 index 0000000000..b66c0fa398 --- /dev/null +++ b/backend/core/src/main/resources/config/liquibase/13-38-0/20260713-add-reindex-run-total-count.xml @@ -0,0 +1,30 @@ + + + + + + + + + + + + diff --git a/backend/core/src/main/resources/config/liquibase/13-38-0/20260713-backfill-modified-on.xml b/backend/core/src/main/resources/config/liquibase/13-38-0/20260713-backfill-modified-on.xml new file mode 100644 index 0000000000..8efbba3d0a --- /dev/null +++ b/backend/core/src/main/resources/config/liquibase/13-38-0/20260713-backfill-modified-on.xml @@ -0,0 +1,31 @@ + + + + + + + + + modified_on IS NULL + + + + diff --git a/backend/core/src/main/resources/config/liquibase/13-38-0/20260717-add-prune-progress.xml b/backend/core/src/main/resources/config/liquibase/13-38-0/20260717-add-prune-progress.xml new file mode 100644 index 0000000000..9aba6480be --- /dev/null +++ b/backend/core/src/main/resources/config/liquibase/13-38-0/20260717-add-prune-progress.xml @@ -0,0 +1,36 @@ + + + + + + + + + + + + + + + + + + diff --git a/backend/core/src/main/resources/config/liquibase/changelog-master.xml b/backend/core/src/main/resources/config/liquibase/changelog-master.xml index e8c46e8c69..68ca8fe1a3 100644 --- a/backend/core/src/main/resources/config/liquibase/changelog-master.xml +++ b/backend/core/src/main/resources/config/liquibase/changelog-master.xml @@ -33,5 +33,6 @@ + diff --git a/backend/core/src/main/resources/config/liquibase/initial-setup/initial-setup-master.xml b/backend/core/src/main/resources/config/liquibase/initial-setup/initial-setup-master.xml index b8c359c9c6..21ac06820a 100644 --- a/backend/core/src/main/resources/config/liquibase/initial-setup/initial-setup-master.xml +++ b/backend/core/src/main/resources/config/liquibase/initial-setup/initial-setup-master.xml @@ -1,6 +1,6 @@ + +
+ + + {{ 'adminSettings.opensearch.reindex.title' | translate }} + + +
+ +
+ + + + + +
+ + +
+ {{ 'adminSettings.opensearch.reindex.started' | translate }} + {{ data.startedOn | date:'medium' }} + + {{ 'adminSettings.opensearch.reindex.finished' | translate }} + {{ data.finishedOn ? (data.finishedOn | date:'medium') : '-' }} + + {{ 'adminSettings.opensearch.reindex.documentDefinitionName' | translate }} + {{ data.scope?.documentDefinitionName || '-' }} + + {{ 'adminSettings.opensearch.reindex.pruneOrphans' | translate }} + + {{ data.scope?.pruneOrphans ? ('interface.yes' | translate) : ('interface.no' | translate) }} + + + + {{ 'adminSettings.opensearch.reindex.modifiedAfter' | translate }} + {{ data.scope?.modifiedAfter ? (data.scope.modifiedAfter | date:'medium') : '-' }} + +
+
+ + {{ 'adminSettings.opensearch.reindex.reindexing' | translate }} + + +
+ {{ 'adminSettings.opensearch.reindex.processed' | translate:{current: data.processedCount, total: data.totalCount} }} + {{ 'adminSettings.opensearch.reindex.skipped' | translate }}: {{ data.skippedCount || 0 }} +
+
+ + @if (data.pruningPhase || data.pruneTotalCount) { +
+ + {{ 'adminSettings.opensearch.reindex.pruning' | translate }} + + + +
+ {{ 'adminSettings.opensearch.reindex.processed' | translate:{current: data.pruneCheckedCount, total: data.pruneTotalCount || 0} }} + {{ 'adminSettings.opensearch.reindex.pruningProgress' | translate:{removed: data.prunedCount || 0} }} +
+
+ } + + @if (data.status === 'FAILED') { + + {{ 'adminSettings.opensearch.reindex.viewErrorLogs' | translate }} + + + } +
+
+ diff --git a/frontend/projects/valtimo/admin-settings/src/lib/components/admin-settings-opensearch/admin-settings-opensearch.component.scss b/frontend/projects/valtimo/admin-settings/src/lib/components/admin-settings-opensearch/admin-settings-opensearch.component.scss new file mode 100644 index 0000000000..1ae611589a --- /dev/null +++ b/frontend/projects/valtimo/admin-settings/src/lib/components/admin-settings-opensearch/admin-settings-opensearch.component.scss @@ -0,0 +1,136 @@ +/*! + * Copyright 2015-2026 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +.opensearch-reindex { + width: 100%; + display: flex; + flex-direction: column; + gap: 24px; + + // Fixed table layout so the expanded detail row can align its columns with the + // parent table columns: [expand] [status] [started] [finished] [progress]. + // The expand column is pinned to 3.5rem and each data column is set to 25%. + // Carbon renders the expanded cell with colspan = dataColumns + 2, which adds a + // phantom trailing column; giving the four columns 25% each (100% total, which + // together with the fixed expand column over-claims the width) forces the + // browser to scale them to fill the row exactly and collapse that phantom + // column to zero — so the four data columns fill the full width in equal + // quarters that the expanded grid (repeat(4, 1fr)) mirrors precisely. + ::ng-deep .cds--data-table { + table-layout: fixed; + + th.cds--table-expand { + width: 3.5rem; + } + + thead th:not(.cds--table-expand) { + width: 25%; + } + } + + // Align the expanded detail cell so its content starts at the status column + // (3.5rem = expand-column width) and spans the full remaining table width. + ::ng-deep tr[data-child-row] > td { + padding-inline: 3.5rem 0; + } + + &__expanded { + padding: 8px 0 16px; + display: grid; + // Four equal columns mirroring the (phantom-collapsed) table columns: + // 1 = status, 2 = started, 3 = finished, 4 = progress. Cell content is inset + // by 1rem (matching Carbon's cell padding) so it lines up with the headers. + grid-template-columns: repeat(4, 1fr); + column-gap: 0; + row-gap: 8px; + align-items: start; + } + + &__label { + grid-column: 1; + padding-inline: 1rem; + font-size: 14px; + font-weight: 600; + color: var(--cds-text-primary); + } + + &__value { + grid-column: 2; + padding-inline: 1rem; + font-size: 14px; + color: var(--cds-text-primary); + } + + // Progress bars align with "finished" and span the finished + progress columns. + // grid-row end must match the number of label/value rows (5) so the column + // spans them exactly — a larger value adds empty rows whose row-gap leaves + // dead space below the last label. + &__results-column { + grid-column: 3 / span 2; + grid-row: 1 / 6; + padding-inline: 1rem; + display: flex; + flex-direction: column; + gap: 16px; + } + + &__value-with-tooltip { + display: inline-flex; + align-items: center; + gap: 4px; + } + + &__progress-label { + display: inline-flex; + align-items: center; + gap: 4px; + margin-bottom: 4px; + font-size: 14px; + font-weight: 600; + color: var(--cds-text-secondary); + } + + &__progress-footer { + display: flex; + justify-content: space-between; + align-items: center; + margin-top: 4px; + font-size: 12px; + color: var(--cds-text-secondary); + } + + &__progress { + padding: 8px 0; + + ::ng-deep .cds--progress-bar__track { + background-color: var(--cds-border-subtle); + } + } + + &__error-link { + display: inline-flex; + align-items: center; + gap: 4px; + color: var(--cds-link-primary); + font-size: 14px; + cursor: pointer; + text-decoration: none; + + &:hover { + text-decoration: underline; + } + } +} diff --git a/frontend/projects/valtimo/admin-settings/src/lib/components/admin-settings-opensearch/admin-settings-opensearch.component.ts b/frontend/projects/valtimo/admin-settings/src/lib/components/admin-settings-opensearch/admin-settings-opensearch.component.ts new file mode 100644 index 0000000000..9defeaef45 --- /dev/null +++ b/frontend/projects/valtimo/admin-settings/src/lib/components/admin-settings-opensearch/admin-settings-opensearch.component.ts @@ -0,0 +1,246 @@ +/* + * Copyright 2015-2026 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import {ChangeDetectionStrategy, Component, OnDestroy, OnInit} from '@angular/core'; +import {CommonModule, DatePipe} from '@angular/common'; +import {Router} from '@angular/router'; +import {TranslateModule, TranslateService} from '@ngx-translate/core'; +import { + BehaviorSubject, + finalize, + interval, + map, + merge, + Observable, + of, + shareReplay, + startWith, + Subject, + switchMap, + take, + takeUntil, + takeWhile, +} from 'rxjs'; +import { + ButtonModule, + IconModule, + IconService, + ListItem, + LoadingModule, + ProgressBarModule, + TagModule, +} from 'carbon-components-angular'; +import {Launch16} from '@carbon/icons'; +import {CarbonListModule, ColumnConfig, Pagination, TooltipIconModule, ViewType} from '@valtimo/components'; +import {Page} from '@valtimo/shared'; +import {AdminSettingsManagementApiService} from '../../services'; +import {ReindexStatusDto, StartReindexRequestDto} from '../../models'; +import {DocumentService} from '@valtimo/document'; +import {StartReindexModalComponent} from '../start-reindex-modal/start-reindex-modal.component'; + +@Component({ + standalone: true, + selector: 'valtimo-admin-settings-opensearch', + templateUrl: './admin-settings-opensearch.component.html', + styleUrls: ['./admin-settings-opensearch.component.scss'], + imports: [ + CommonModule, + DatePipe, + TranslateModule, + ButtonModule, + IconModule, + LoadingModule, + ProgressBarModule, + TagModule, + CarbonListModule, + StartReindexModalComponent, + TooltipIconModule, + ], + changeDetection: ChangeDetectionStrategy.OnPush, +}) +export class AdminSettingsOpensearchComponent implements OnInit, OnDestroy { + private readonly _destroy$ = new Subject(); + private readonly _manualRefresh$ = new BehaviorSubject(undefined); + private readonly _silentRefresh$ = new Subject(); + + public readonly fields: ColumnConfig[] = [ + {key: 'statusTag', label: 'adminSettings.opensearch.reindex.columns.status', viewType: ViewType.TAGS}, + {key: 'startedOn', label: 'adminSettings.opensearch.reindex.columns.startedOn', viewType: ViewType.DATE}, + {key: 'finishedOn', label: 'adminSettings.opensearch.reindex.columns.finishedOn', viewType: ViewType.DATE}, + {key: 'progress', label: 'adminSettings.opensearch.reindex.columns.progress', viewType: ViewType.TEXT}, + ]; + + public pagination: Pagination = { + collectionSize: 0, + page: 1, + size: 10, + }; + + public readonly showModal$ = new BehaviorSubject(false); + public readonly startingReindex$ = new BehaviorSubject(false); + public readonly loading$ = new BehaviorSubject(false); + + public documentDefinitions$: Observable; + + public readonly runs$: Observable> = merge( + this._manualRefresh$.pipe( + switchMap(() => { + this.loading$.next(true); + return this._apiService.getReindexRuns(this.pagination.page - 1, this.pagination.size).pipe( + finalize(() => this.loading$.next(false)) + ); + }) + ), + this._silentRefresh$.pipe( + switchMap(() => this._apiService.getReindexRuns(this.pagination.page - 1, this.pagination.size)) + ) + ).pipe(shareReplay(1)); + + public readonly tableItems$: Observable = this.runs$.pipe( + switchMap(page => { + this.pagination = {...this.pagination, collectionSize: page.totalElements}; + if (page.content.length === 0) { + return of([]); + } + const statusKeys = page.content.map(run => `adminSettings.opensearch.reindex.statuses.${run.status}`); + return this._translateService.get(statusKeys).pipe( + map(translations => page.content.map(run => ({ + ...run, + progress: `${run.processedCount} / ${run.totalCount}`, + statusTag: { + content: translations[`adminSettings.opensearch.reindex.statuses.${run.status}`], + type: this._getStatusTagType(run.status), + }, + }))) + ); + }) + ); + + public readonly hasRunningRun$: Observable = this.runs$.pipe( + map(page => page.content.some(run => run.status === 'RUNNING')) + ); + + constructor( + private readonly _apiService: AdminSettingsManagementApiService, + private readonly _documentService: DocumentService, + private readonly _translateService: TranslateService, + private readonly _router: Router, + private readonly _iconService: IconService + ) { + this._iconService.register(Launch16); + } + + public ngOnInit(): void { + this.documentDefinitions$ = this._documentService.queryDefinitionsForManagement().pipe( + switchMap(page => + this._translateService.get('adminSettings.opensearch.reindex.allDocumentDefinitions').pipe( + map(allLabel => [ + {content: allLabel, value: null, selected: false}, + ...page.content.map(def => ({ + content: def.id.name, + value: def.id.name, + selected: false, + })), + ]) + ) + ), + startWith([]) + ); + + this.hasRunningRun$ + .pipe( + switchMap(hasRunning => { + if (!hasRunning) return []; + return interval(3000).pipe( + takeWhile(() => true), + takeUntil(this._destroy$) + ); + }), + takeUntil(this._destroy$) + ) + .subscribe(() => this._silentRefresh$.next()); + } + + public onPageChange(page: number): void { + this.pagination = {...this.pagination, page}; + this._manualRefresh$.next(); + } + + public onPageSizeChange(size: number): void { + this.pagination = {...this.pagination, size, page: 1}; + this._manualRefresh$.next(); + } + + public openModal(): void { + this.showModal$.next(true); + } + + public onModalClose(request: StartReindexRequestDto | null): void { + this.showModal$.next(false); + if (!request) return; + + this.startingReindex$.next(true); + this._apiService + .startReindex(request) + .pipe( + take(1), + finalize(() => this.startingReindex$.next(false)) + ) + .subscribe({ + next: () => this._manualRefresh$.next(), + error: err => { + if (err.status === 409) { + this._manualRefresh$.next(); + } + }, + }); + } + + private _getStatusTagType(status: string): string { + switch (status) { + case 'RUNNING': + return 'blue'; + case 'COMPLETED': + return 'green'; + case 'FAILED': + return 'red'; + case 'STOPPED': + return 'gray'; + default: + return 'gray'; + } + } + + public navigateToLogs(run: ReindexStatusDto): void { + const afterTimestamp = new Date(new Date(run.startedOn).getTime() - 5000).toISOString(); + const beforeTimestamp = run.finishedOn + ? new Date(new Date(run.finishedOn).getTime() + 5000).toISOString() + : new Date().toISOString(); + + this._router.navigate(['/logging'], { + queryParams: { + level: 'ERROR', + afterTimestamp, + beforeTimestamp, + }, + }); + } + + public ngOnDestroy(): void { + this._destroy$.next(); + this._destroy$.complete(); + } +} diff --git a/frontend/projects/valtimo/admin-settings/src/lib/components/admin-settings/admin-settings.component.html b/frontend/projects/valtimo/admin-settings/src/lib/components/admin-settings/admin-settings.component.html index 4caeb3d201..3988fabadf 100644 --- a/frontend/projects/valtimo/admin-settings/src/lib/components/admin-settings/admin-settings.component.html +++ b/frontend/projects/valtimo/admin-settings/src/lib/components/admin-settings/admin-settings.component.html @@ -1,17 +1,19 @@ @if (activeTabKey$ | async; as activeTabKey) { @@ -39,5 +41,6 @@ } } + } diff --git a/frontend/projects/valtimo/admin-settings/src/lib/components/start-reindex-modal/start-reindex-modal.component.html b/frontend/projects/valtimo/admin-settings/src/lib/components/start-reindex-modal/start-reindex-modal.component.html new file mode 100644 index 0000000000..400035fa4a --- /dev/null +++ b/frontend/projects/valtimo/admin-settings/src/lib/components/start-reindex-modal/start-reindex-modal.component.html @@ -0,0 +1,55 @@ + + + + +

{{ 'adminSettings.opensearch.reindex.startModalTitle' | translate }}

+ + +
+ + + {{ 'adminSettings.opensearch.reindex.pruneOrphans' | translate }} + + + + + + + + + +
+ + + + + + + diff --git a/frontend/projects/valtimo/admin-settings/src/lib/components/start-reindex-modal/start-reindex-modal.component.scss b/frontend/projects/valtimo/admin-settings/src/lib/components/start-reindex-modal/start-reindex-modal.component.scss new file mode 100644 index 0000000000..1798d7b259 --- /dev/null +++ b/frontend/projects/valtimo/admin-settings/src/lib/components/start-reindex-modal/start-reindex-modal.component.scss @@ -0,0 +1,27 @@ +/*! + * Copyright 2015-2026 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +.start-reindex-form { + display: flex; + flex-direction: column; + gap: 1rem; + + .label-with-tooltip { + display: inline-flex; + align-items: center; + gap: 4px; + } +} diff --git a/frontend/projects/valtimo/admin-settings/src/lib/components/start-reindex-modal/start-reindex-modal.component.ts b/frontend/projects/valtimo/admin-settings/src/lib/components/start-reindex-modal/start-reindex-modal.component.ts new file mode 100644 index 0000000000..be036db887 --- /dev/null +++ b/frontend/projects/valtimo/admin-settings/src/lib/components/start-reindex-modal/start-reindex-modal.component.ts @@ -0,0 +1,126 @@ +/* + * Copyright 2015-2026 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import {ChangeDetectionStrategy, Component, EventEmitter, Input, Output} from '@angular/core'; +import {CommonModule} from '@angular/common'; +import {FormBuilder, FormGroup, ReactiveFormsModule} from '@angular/forms'; +import {TranslateModule} from '@ngx-translate/core'; +import { + ButtonModule, + CheckboxModule, + DatePickerInputModule, + DatePickerModule, + DropdownModule, + LayerModule, + ListItem, + ModalModule, +} from 'carbon-components-angular'; +import {TooltipIconModule, ValtimoCdsModalDirective} from '@valtimo/components'; +import {StartReindexRequestDto} from '../../models'; + +@Component({ + standalone: true, + selector: 'valtimo-start-reindex-modal', + templateUrl: './start-reindex-modal.component.html', + styleUrls: ['./start-reindex-modal.component.scss'], + imports: [ + CommonModule, + TranslateModule, + ReactiveFormsModule, + ModalModule, + ValtimoCdsModalDirective, + ButtonModule, + CheckboxModule, + DatePickerInputModule, + DatePickerModule, + DropdownModule, + LayerModule, + TooltipIconModule, + ], + changeDetection: ChangeDetectionStrategy.OnPush, +}) +export class StartReindexModalComponent { + @Input() public open = false; + @Input() public documentDefinitions: ListItem[] = []; + + @Output() public readonly closeEvent = new EventEmitter(); + + public readonly formGroup: FormGroup = this._fb.group({ + pruneOrphans: [false], + documentDefinitionName: [null], + modifiedAfter: [null], + }); + + constructor(private readonly _fb: FormBuilder) {} + + public onDateSelected(event: string[]): void { + const dateValue = event?.[0] || null; + this.formGroup.patchValue({modifiedAfter: dateValue}); + } + + public onCancel(): void { + this._resetForm(); + this.closeEvent.emit(null); + } + + public onSubmit(): void { + const request = this._buildRequest(); + this._resetForm(); + this.closeEvent.emit(request); + } + + private _buildRequest(): StartReindexRequestDto { + const formValue = this.formGroup.value; + const request: StartReindexRequestDto = {}; + + if (formValue.pruneOrphans) { + request.pruneOrphans = true; + } + + if (formValue.documentDefinitionName?.value) { + request.documentDefinitionName = formValue.documentDefinitionName.value; + } + + if (formValue.modifiedAfter) { + request.modifiedAfter = this._formatDateToIso(formValue.modifiedAfter); + } + + return request; + } + + private _formatDateToIso(date: string | Date): string { + if (date instanceof Date) { + const year = date.getFullYear(); + const month = String(date.getMonth() + 1).padStart(2, '0'); + const day = String(date.getDate()).padStart(2, '0'); + return `${year}-${month}-${day}T00:00:00`; + } + const parts = date.split('-'); + if (parts.length === 3) { + const [day, month, year] = parts; + return `${year}-${month}-${day}T00:00:00`; + } + return date; + } + + private _resetForm(): void { + this.formGroup.reset({ + pruneOrphans: false, + documentDefinitionName: null, + modifiedAfter: null, + }); + } +} diff --git a/frontend/projects/valtimo/admin-settings/src/lib/constants/feature-toggle.constants.ts b/frontend/projects/valtimo/admin-settings/src/lib/constants/feature-toggle.constants.ts index f48853271e..3a2b0af449 100644 --- a/frontend/projects/valtimo/admin-settings/src/lib/constants/feature-toggle.constants.ts +++ b/frontend/projects/valtimo/admin-settings/src/lib/constants/feature-toggle.constants.ts @@ -1,17 +1,19 @@ /* - * Copyright 2015-2026 Ritense BV, the Netherlands. * - * Licensed under EUPL, Version 1.2 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * * Copyright 2015-2026 Ritense BV, the Netherlands. + * * + * * Licensed under EUPL, Version 1.2 (the "License"); + * * you may not use this file except in compliance with the License. + * * You may obtain a copy of the License at + * * + * * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" basis, + * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * * See the License for the specific language governing permissions and + * * limitations under the License. * - * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" basis, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. */ import {FeatureToggleDefinition} from '../models'; @@ -39,6 +41,7 @@ const FEATURE_TOGGLE_DEFINITIONS: FeatureToggleDefinition[] = [ {key: 'enableSuppressDocumentError'}, {key: 'enableIkoType'}, {key: 'menuCollapsedByDefault'}, + {key: 'enableGenericCaseList'}, ]; export {FEATURE_TOGGLE_DEFINITIONS}; diff --git a/frontend/projects/valtimo/admin-settings/src/lib/models/index.ts b/frontend/projects/valtimo/admin-settings/src/lib/models/index.ts index af992acfb5..d70ca73577 100644 --- a/frontend/projects/valtimo/admin-settings/src/lib/models/index.ts +++ b/frontend/projects/valtimo/admin-settings/src/lib/models/index.ts @@ -1,18 +1,22 @@ /* - * Copyright 2015-2026 Ritense BV, the Netherlands. * - * Licensed under EUPL, Version 1.2 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * * Copyright 2015-2026 Ritense BV, the Netherlands. + * * + * * Licensed under EUPL, Version 1.2 (the "License"); + * * you may not use this file except in compliance with the License. + * * You may obtain a copy of the License at + * * + * * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" basis, + * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * * See the License for the specific language governing permissions and + * * limitations under the License. * - * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" basis, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. */ export * from './accent-colors.model'; export * from './feature-toggle.model'; +export * from './reindex.model'; +export * from './search-engine.model'; diff --git a/frontend/projects/valtimo/admin-settings/src/lib/models/reindex.model.ts b/frontend/projects/valtimo/admin-settings/src/lib/models/reindex.model.ts new file mode 100644 index 0000000000..c9d73c8514 --- /dev/null +++ b/frontend/projects/valtimo/admin-settings/src/lib/models/reindex.model.ts @@ -0,0 +1,64 @@ +/* + * Copyright 2015-2026 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +type ReindexStatus = 'RUNNING' | 'COMPLETED' | 'FAILED' | 'STOPPED'; + +interface ReindexScope { + pruneOrphans?: boolean; + documentDefinitionName?: string; + modifiedBefore?: string; + modifiedAfter?: string; + pageSize?: number; + resumeRunId?: string; + documentIds?: string[]; +} + +interface ReindexStatusDto { + runId: string; + status: ReindexStatus; + running: boolean; + scope: ReindexScope | null; + totalCount: number; + processedCount: number; + skippedCount: number; + prunedCount: number; + pruneCheckedCount: number; + pruneTotalCount: number | null; + pruningPhase: boolean; + startedOn: string; + finishedOn: string | null; + elapsedSeconds: number; + error: string | null; +} + +interface StartReindexRequestDto { + pruneOrphans?: boolean; + documentDefinitionName?: string; + modifiedAfter?: string; +} + +interface StartReindexResponseDto { + status: string; + runId: string; +} + +export { + ReindexStatus, + ReindexScope, + ReindexStatusDto, + StartReindexRequestDto, + StartReindexResponseDto, +}; diff --git a/frontend/projects/valtimo/admin-settings/src/lib/models/search-engine.model.ts b/frontend/projects/valtimo/admin-settings/src/lib/models/search-engine.model.ts new file mode 100644 index 0000000000..c0e6b47c24 --- /dev/null +++ b/frontend/projects/valtimo/admin-settings/src/lib/models/search-engine.model.ts @@ -0,0 +1,24 @@ +/* + * Copyright 2015-2026 Ritense BV, the Netherlands. + * + * Licensed under EUPL, Version 1.2 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" basis, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export interface SearchEngineDto { + available: boolean; + active: 'OPENSEARCH' | 'POSTGRES'; +} + +export interface UpdateSearchEngineDto { + active: 'OPENSEARCH' | 'POSTGRES'; +} diff --git a/frontend/projects/valtimo/admin-settings/src/lib/services/admin-settings-management-api.service.ts b/frontend/projects/valtimo/admin-settings/src/lib/services/admin-settings-management-api.service.ts index ee6c45e0ac..d881a8e6ea 100644 --- a/frontend/projects/valtimo/admin-settings/src/lib/services/admin-settings-management-api.service.ts +++ b/frontend/projects/valtimo/admin-settings/src/lib/services/admin-settings-management-api.service.ts @@ -15,16 +15,25 @@ */ import {Injectable} from '@angular/core'; -import {HttpClient} from '@angular/common/http'; -import {Observable} from 'rxjs'; +import {HttpClient, HttpParams} from '@angular/common/http'; +import {catchError, Observable, of} from 'rxjs'; import { AdminSettingsLogoDto, AdminSettingsLogosDto, BaseApiService, ConfigService, CreateAdminSettingsLogoDto, + Page, } from '@valtimo/shared'; -import {AccentColorsDto, FeatureToggleOverridesDto, UpdateFeatureToggleDto} from '../models'; +import { + AccentColorsDto, + FeatureToggleOverridesDto, + ReindexStatusDto, + SearchEngineDto, + StartReindexRequestDto, + StartReindexResponseDto, + UpdateFeatureToggleDto, +} from '../models'; @Injectable({ providedIn: 'root', @@ -90,4 +99,44 @@ export class AdminSettingsManagementApiService extends BaseApiService { dto ); } + + public getSearchEngine(): Observable { + return this.httpClient + .get(this.getApiUrl('/management/v1/search-engine')) + .pipe(catchError(() => of(null))); + } + + public updateSearchEngine(useOpenSearch: boolean): Observable { + return this.httpClient.put( + this.getApiUrl('/management/v1/search-engine'), + {active: useOpenSearch ? 'OPENSEARCH' : 'POSTGRES'} + ); + } + + public startReindex(request: StartReindexRequestDto = {}): Observable { + return this.httpClient.post( + this.getApiUrl('/management/v1/document-opensearch/reindex'), + request + ); + } + + public getReindexStatus(): Observable { + return this.httpClient + .get(this.getApiUrl('/management/v1/document-opensearch/reindex/status')) + .pipe(catchError(() => of(null))); + } + + public getReindexRuns(page: number, size: number): Observable> { + const params = new HttpParams().set('page', page.toString()).set('size', size.toString()); + return this.httpClient.get>( + this.getApiUrl('/management/v1/document-opensearch/reindex/runs'), + {params} + ); + } + + public getReindexRun(runId: string): Observable { + return this.httpClient.get( + this.getApiUrl(`/management/v1/document-opensearch/reindex/${runId}`) + ); + } } diff --git a/frontend/projects/valtimo/bootstrap/src/lib/init.ts b/frontend/projects/valtimo/bootstrap/src/lib/init.ts index 696ecce0fd..a495aeddf6 100644 --- a/frontend/projects/valtimo/bootstrap/src/lib/init.ts +++ b/frontend/projects/valtimo/bootstrap/src/lib/init.ts @@ -1,23 +1,26 @@ /* - * Copyright 2015-2025 Ritense BV, the Netherlands. * - * Licensed under EUPL, Version 1.2 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * * Copyright 2015-2026 Ritense BV, the Netherlands. + * * + * * Licensed under EUPL, Version 1.2 (the "License"); + * * you may not use this file except in compliance with the License. + * * You may obtain a copy of the License at + * * + * * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" basis, + * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * * See the License for the specific language governing permissions and + * * limitations under the License. * - * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" basis, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. */ import {NGXLogger} from 'ngx-logger'; import {TranslateService} from '@ngx-translate/core'; import {accountInitializer} from '@valtimo/account'; import {Injector} from '@angular/core'; +import {HttpClient} from '@angular/common/http'; import {ConfigService} from '@valtimo/shared'; import {AdminSettingsService, menuInitializer} from '@valtimo/components'; import {firstValueFrom} from 'rxjs'; @@ -88,6 +91,23 @@ export function initializerFactory( } }); + // Check OpenSearch availability and patch feature toggle + initializersArray.push(async () => { + try { + const httpClient = injector.get(HttpClient); + const response = await firstValueFrom( + httpClient.get<{available: boolean}>( + `${configService.config.valtimoApi.endpointUri}management/v1/search-engine` + ) + ); + if (response?.available) { + configService.patchFeatureToggles({enableOpenSearch: true}); + } + } catch { + // OpenSearch not available + } + }); + // Use environment config initializers to be used in app startup. configService.initializers.forEach(initializer => { initializersArray.push(initializer(injector)); diff --git a/frontend/projects/valtimo/case/src/lib/components/case-list/case-list.component.html b/frontend/projects/valtimo/case/src/lib/components/case-list/case-list.component.html index 295b2364d6..4e2bb64f01 100644 --- a/frontend/projects/valtimo/case/src/lib/components/case-list/case-list.component.html +++ b/frontend/projects/valtimo/case/src/lib/components/case-list/case-list.component.html @@ -39,6 +39,7 @@ assigneeFilter: orchestration.assigneeFilter$ | async, hiddenColumns: orchestration.hiddenColumns$ | async, disableStartButton: disableStartButton$ | async, + invalidSearchFields: orchestration.invalidSearchFields$ | async, } as obs" > diff --git a/frontend/projects/valtimo/case/src/lib/components/case-list/case-list.component.ts b/frontend/projects/valtimo/case/src/lib/components/case-list/case-list.component.ts index 28d7a22c02..6ae57fe2f7 100644 --- a/frontend/projects/valtimo/case/src/lib/components/case-list/case-list.component.ts +++ b/frontend/projects/valtimo/case/src/lib/components/case-list/case-list.component.ts @@ -1,5 +1,5 @@ /* - * Copyright 2015-2025 Ritense BV, the Netherlands. + * Copyright 2015-2026 Ritense BV, the Netherlands. * * Licensed under EUPL, Version 1.2 (the "License"); * you may not use this file except in compliance with the License. @@ -151,6 +151,10 @@ export class CaseListComponent implements OnInit, OnDestroy { this.searchService.search(searchFieldValues); } + public onGlobalSearchFilterChange(value: string): void { + this.searchService.setGlobalSearchFilter(value); + } + // --- Row click --- public rowClick(item: any): void { @@ -298,6 +302,7 @@ export class CaseListComponent implements OnInit, OnDestroy { this.parameterService.setSearchFieldValues( this.parameterService.getSearchObject(queryParams['search']) as SearchFieldValues ); + this.searchService.setGlobalSearchFilter(null); }); } diff --git a/frontend/projects/valtimo/case/src/lib/components/generic-case-list/generic-case-list.component.html b/frontend/projects/valtimo/case/src/lib/components/generic-case-list/generic-case-list.component.html index e83b04ca29..f35341b6c5 100644 --- a/frontend/projects/valtimo/case/src/lib/components/generic-case-list/generic-case-list.component.html +++ b/frontend/projects/valtimo/case/src/lib/components/generic-case-list/generic-case-list.component.html @@ -1,17 +1,19 @@ @@ -20,6 +22,9 @@ diff --git a/frontend/projects/valtimo/case/src/lib/components/generic-case-list/generic-case-list.component.ts b/frontend/projects/valtimo/case/src/lib/components/generic-case-list/generic-case-list.component.ts index 422e300b49..400e076dc7 100644 --- a/frontend/projects/valtimo/case/src/lib/components/generic-case-list/generic-case-list.component.ts +++ b/frontend/projects/valtimo/case/src/lib/components/generic-case-list/generic-case-list.component.ts @@ -150,7 +150,10 @@ export class GenericCaseListComponent implements OnInit, OnDestroy { private readonly _allCasesSort$ = new BehaviorSubject(null); private readonly _allCasesReload$ = new BehaviorSubject(false); private readonly _allCasesAssigneeFilter$ = new BehaviorSubject('ALL'); + private readonly _allCasesGlobalSearch$ = new BehaviorSubject(''); public readonly allCasesAssigneeFilter$ = this._allCasesAssigneeFilter$.asObservable(); + public readonly allCasesGlobalSearch$ = this._allCasesGlobalSearch$.asObservable(); + public readonly allCasesInvalidSearchFields$ = this.documentService.invalidSearchFields$; public readonly allCasesFields$: Observable = this.translateService .stream('fieldLabels') @@ -170,16 +173,17 @@ export class GenericCaseListComponent implements OnInit, OnDestroy { this._allCasesSort$, this._allCasesReload$, this._allCasesAssigneeFilter$, + this._allCasesGlobalSearch$, ]).pipe( tap(() => this.allCasesLoading$.next(true)), - switchMap(([page, size, sort, _, assigneeFilter]) => { + switchMap(([page, size, sort, _, assigneeFilter, globalSearch]) => { const request = new DocumentSearchRequestImpl( '', page - 1, size, undefined, undefined, - undefined, + globalSearch || undefined, sort, undefined, assigneeFilter !== 'ALL' ? assigneeFilter : undefined @@ -220,7 +224,7 @@ export class GenericCaseListComponent implements OnInit, OnDestroy { private readonly parameterService: CaseParameterService, private readonly quickSearchStateService: QuickSearchStateService, private readonly router: Router, - private readonly searchService: CaseListSearchService, + public readonly searchService: CaseListSearchService, private readonly statusService: CaseListStatusService, @Inject(QUICK_SEARCH_SERVICE) private readonly caseListQuickSearchService: IQuickSearchService, @@ -260,12 +264,14 @@ export class GenericCaseListComponent implements OnInit, OnDestroy { this.parameterService.clearSearchFieldValues(); this.paginationService.clearPagination(); this.assigneeService.resetAssigneeFilter(); + this.searchService.setGlobalSearchFilter(null); this.listService.setCaseDefinitionKey(newId); this.orchestration.setLoading(); this.subscribeToPagination(); this.subscribeToCanHaveAssignee(); this.subscribeToSearchFields(); } else { + this._allCasesGlobalSearch$.next(''); this._allCasesPage$.next(1); this._allCasesReload$.next(!this._allCasesReload$.getValue()); } @@ -412,6 +418,7 @@ export class GenericCaseListComponent implements OnInit, OnDestroy { this.parameterService.setSearchFieldValues( this.parameterService.getSearchObject(queryParams['search']) as SearchFieldValues ); + this.searchService.setGlobalSearchFilter(null); }); } @@ -440,6 +447,15 @@ export class GenericCaseListComponent implements OnInit, OnDestroy { this._allCasesPage$.next(1); } + public allCasesSearch(searchTerm: string): void { + this._allCasesGlobalSearch$.next(searchTerm); + this._allCasesPage$.next(1); + } + + public onGlobalSearchFilterChange(value: string): void { + this.searchService.setGlobalSearchFilter(value); + } + // --- Private --- private subscribeToPagination(): void { diff --git a/frontend/projects/valtimo/case/src/lib/services/case-list-orchestration.service.ts b/frontend/projects/valtimo/case/src/lib/services/case-list-orchestration.service.ts index 72048ae4d4..0cf0728b0b 100644 --- a/frontend/projects/valtimo/case/src/lib/services/case-list-orchestration.service.ts +++ b/frontend/projects/valtimo/case/src/lib/services/case-list-orchestration.service.ts @@ -114,6 +114,11 @@ export class CaseListOrchestrationService { public readonly searchFields$: Observable | null> = this.searchService.documentSearchFields$; + public readonly globalSearchFilter$: Observable = this.searchService.globalSearchFilter$; + + public readonly invalidSearchFields$: Observable = + this.documentService.invalidSearchFields$; + public readonly statuses$: Observable> = this.statusService.caseStatuses$; @@ -330,6 +335,7 @@ export class CaseListOrchestrationService { this.hasApiColumnConfig$, this.statusService.caseStatuses$, this.caseListCaseTagService.caseTags$, + this.globalSearchFilter$, ]).pipe(debounceTime(50)) ), distinctUntilChanged(this.areDocumentRequestsEqual), @@ -343,6 +349,8 @@ export class CaseListOrchestrationService { _, hasApiColumnConfig, allStatuses, + __, + globalSearchFilter, ]) => this.fetchDocuments( documentSearchRequest, @@ -351,7 +359,8 @@ export class CaseListOrchestrationService { selectedStatuses, selectedCaseTagKeys, hasApiColumnConfig, - allStatuses + allStatuses, + globalSearchFilter ) ), switchMap(res => this.checkDocumentPermissions(res)), @@ -439,6 +448,10 @@ export class CaseListOrchestrationService { prevSelectedStatuses, prevCaseTagKeys, prevForceRefresh, + _prevHasApiColumnConfig, + _prevStatuses, + _prevCaseTags, + prevGlobalSearchFilter, ]: any[], [ currSearchRequest, @@ -447,6 +460,10 @@ export class CaseListOrchestrationService { currSelectedStatuses, currCaseTagKeys, currForceRefresh, + _currHasApiColumnConfig, + _currStatuses, + _currCaseTags, + currGlobalSearchFilter, ]: any[] ): boolean { return isEqual( @@ -457,6 +474,7 @@ export class CaseListOrchestrationService { ...prevSelectedStatuses, ...prevCaseTagKeys, forceRefresh: prevForceRefresh, + globalSearchFilter: prevGlobalSearchFilter, }, { ...currSearchRequest, @@ -465,6 +483,7 @@ export class CaseListOrchestrationService { ...currSelectedStatuses, ...currCaseTagKeys, forceRefresh: currForceRefresh, + globalSearchFilter: currGlobalSearchFilter, } ); } @@ -476,7 +495,8 @@ export class CaseListOrchestrationService { selectedStatuses: string[], selectedCaseTagKeys: string[], hasApiColumnConfig: boolean, - allStatuses: InternalCaseStatus[] + allStatuses: InternalCaseStatus[], + globalSearchFilter?: string ): Observable<{ documents: Documents | SpecifiedDocuments; hasApiColumnConfig: boolean; @@ -496,6 +516,8 @@ export class CaseListOrchestrationService { ? this.searchService.mapSearchValuesToFilters(searchValues) : undefined; + const globalFilter = globalSearchFilter?.trim() || undefined; + const documentsObs = !hasApiColumnConfig ? this.documentService.getDocumentsSearch( documentSearchRequest, @@ -503,7 +525,8 @@ export class CaseListOrchestrationService { assigneeFilter, searchFilters, statusKeys, - selectedCaseTagKeys + selectedCaseTagKeys, + globalFilter ) : this.documentService.getSpecifiedDocumentsSearch( documentSearchRequest, @@ -511,13 +534,14 @@ export class CaseListOrchestrationService { assigneeFilter, searchFilters, statusKeys, - selectedCaseTagKeys + selectedCaseTagKeys, + globalFilter ); return forkJoin({ documents: documentsObs, hasApiColumnConfig: of(hasApiColumnConfig), - isSearchResult: of(!!searchFilters), + isSearchResult: of(!!searchFilters || !!globalFilter), allStatuses: of(allStatuses), assigneeFilter: of(assigneeFilter), }); diff --git a/frontend/projects/valtimo/case/src/lib/services/case-list-search.service.ts b/frontend/projects/valtimo/case/src/lib/services/case-list-search.service.ts index 31130ae518..6db3e7e7f9 100644 --- a/frontend/projects/valtimo/case/src/lib/services/case-list-search.service.ts +++ b/frontend/projects/valtimo/case/src/lib/services/case-list-search.service.ts @@ -15,7 +15,7 @@ */ import {Injectable} from '@angular/core'; -import {Observable, of, switchMap} from 'rxjs'; +import {BehaviorSubject, Observable, of, switchMap} from 'rxjs'; import {SearchField, SearchFieldValues, SearchFilter, SearchFilterRange} from '@valtimo/shared'; import {CaseListService} from './case-list.service'; import {DocumentService} from '@valtimo/document'; @@ -32,16 +32,27 @@ export class CaseListSearchService { ) ); + private readonly _globalSearchFilter$ = new BehaviorSubject(''); + public get documentSearchFields$(): Observable | null> { return this._documentSearchFields$; } + public get globalSearchFilter$(): Observable { + return this._globalSearchFilter$.asObservable(); + } + constructor( private readonly caseListService: CaseListService, private readonly documentService: DocumentService, private readonly caseParameterService: CaseParameterService ) {} + public setGlobalSearchFilter(value: string | null): void { + this._globalSearchFilter$.next(value ?? ''); + this.caseListService.checkRefresh(); + } + public search(searchFieldValues: SearchFieldValues): void { this.caseParameterService.setSearchFieldValues(searchFieldValues || {}); this.caseParameterService.setSearchParameters(searchFieldValues); diff --git a/frontend/projects/valtimo/components/src/lib/components/carbon-list/carbon-list.component.html b/frontend/projects/valtimo/components/src/lib/components/carbon-list/carbon-list.component.html index e19494b2d3..a2cf2ee0b9 100644 --- a/frontend/projects/valtimo/components/src/lib/components/carbon-list/carbon-list.component.html +++ b/frontend/projects/valtimo/components/src/lib/components/carbon-list/carbon-list.component.html @@ -1,17 +1,19 @@ +
+ + +
    +
  • {{ field.title || field.key }}
    • +
+
diff --git a/frontend/projects/valtimo/components/src/lib/components/carbon-list/carbon-list.component.scss b/frontend/projects/valtimo/components/src/lib/components/carbon-list/carbon-list.component.scss index 2d6dfc9935..c0d36930b2 100644 --- a/frontend/projects/valtimo/components/src/lib/components/carbon-list/carbon-list.component.scss +++ b/frontend/projects/valtimo/components/src/lib/components/carbon-list/carbon-list.component.scss @@ -1,5 +1,5 @@ /*! - * Copyright 2015-2025 Ritense BV, the Netherlands. + * Copyright 2015-2026 Ritense BV, the Netherlands. * * Licensed under EUPL, Version 1.2 (the "License"); * you may not use this file except in compliance with the License. @@ -137,4 +137,94 @@ td:first-child { overflow-x: auto; overflow-y: hidden; } + + ::ng-deep .cds--expandable-row:not(.cds--parent-row) td { + border-top: none; + border-bottom-width: 2px; + } + + ::ng-deep tbody .cds--expandable-row.cds--parent-row td { + border-bottom-width: 2px; + } + + ::ng-deep tbody tr:first-child td { + border-top: none !important; + } +} + +.valtimo-search-container { + position: relative; + flex: 1; + + ::ng-deep cds-table-toolbar-search { + display: flex; + justify-content: flex-end; + width: 100%; + + // Expanded: fill full width + .cds--toolbar-search-container-active { + width: 100%; + } + } +} + +.valtimo-search-container ::ng-deep .cds--toolbar-search-container-active input { + color: transparent !important; + caret-color: var(--cds-text-primary, #161616); +} + +.valtimo-search-overlay { + position: absolute; + top: 0; + left: 0; + right: 0; + bottom: 0; + display: flex; + align-items: center; + height: 3rem; + padding: 0 3rem; + font-family: 'IBM Plex Sans', 'Helvetica Neue', Arial, sans-serif; + font-size: 0.875rem; + font-weight: 400; + letter-spacing: 0.16px; + line-height: 1.28572; + color: var(--cds-text-primary, #161616); + white-space: pre; + pointer-events: none; + overflow: hidden; + + &__invalid { + text-decoration: underline wavy var(--cds-support-error, #da1e28); + text-decoration-skip-ink: none; + text-underline-offset: 3px; + } +} + +.valtimo-search-autocomplete { + position: absolute; + top: 100%; + max-height: 150px; + min-width: 120px; + max-width: 250px; + width: auto; + overflow-y: auto; + background: var(--cds-layer); + border: 1px solid var(--cds-border-subtle); + box-shadow: 0 2px 6px rgba(0, 0, 0, 0.1); + z-index: 9000; + list-style: none; + margin: 0; + padding: 0; + + li { + padding: 6px 12px; + cursor: pointer; + font-size: 0.875rem; + white-space: nowrap; + + &:hover, + &.selected { + background: var(--cds-layer-hover); + } + } } diff --git a/frontend/projects/valtimo/components/src/lib/components/carbon-list/carbon-list.component.ts b/frontend/projects/valtimo/components/src/lib/components/carbon-list/carbon-list.component.ts index cf7589a547..e9b4e20ca0 100644 --- a/frontend/projects/valtimo/components/src/lib/components/carbon-list/carbon-list.component.ts +++ b/frontend/projects/valtimo/components/src/lib/components/carbon-list/carbon-list.component.ts @@ -1,5 +1,5 @@ /* - * Copyright 2015-2025 Ritense BV, the Netherlands. + * Copyright 2015-2026 Ritense BV, the Netherlands. * * Licensed under EUPL, Version 1.2 (the "License"); * you may not use this file except in compliance with the License. @@ -16,6 +16,7 @@ import { AfterViewInit, ChangeDetectionStrategy, + ChangeDetectorRef, Component, ElementRef, EventEmitter, @@ -30,6 +31,7 @@ import {FormControl} from '@angular/forms'; import {ArrowDown16, ArrowUp16, Draggable16, SettingsView16} from '@carbon/icons'; import {TranslateService} from '@ngx-translate/core'; import {SortState} from '@valtimo/document'; +import {SearchField} from '@valtimo/shared'; import { IconService, PaginationModel, @@ -178,6 +180,19 @@ export class CarbonListComponent implements OnInit, AfterViewInit, OnDestroy { } @Input() isSearchable = false; + @Input() set initialSearchValue(value: string | null) { + if (this._initialSearchValue !== value) { + this._initialSearchValue = value; + this.searchFormControl.setValue(value || '', {emitEvent: false}); + this._lastExecutedSearch = value; + this.searchActive = !!value; + } + } + private _initialSearchValue: string | null = null; + public searchActive = false; + @Input() searchDebounceMs = 500; + @Input() invalidSearchFields: string[] = []; + @Input() searchFields: SearchField[] = []; @Input() enableSingleSelection = false; /** * @deprecated The lastColumnTemplate field is deprecated. Any template column can be added through the **@Input field**. @@ -191,6 +206,8 @@ export class CarbonListComponent implements OnInit, AfterViewInit, OnDestroy { @Input() movingRowsEnabled: boolean; @Input() dragAndDrop = false; @Input() dragAndDropDisabled = false; + @Input() expandedRowTemplate: TemplateRef; + @Input() expandedRowKey: string; @Output() rowClicked = new EventEmitter(); @Output() paginationClicked = new EventEmitter(); @@ -247,9 +264,16 @@ export class CarbonListComponent implements OnInit, AfterViewInit, OnDestroy { public skeletonModel = Table.skeletonModel(5, 5); public paginationModel: PaginationModel; public searchFormControl = new FormControl(''); + public showAutocomplete = false; + public filteredSuggestions: SearchField[] = []; + public selectedSuggestionIndex = -1; + public autocompleteLeft = 0; + private _lastExecutedSearch: string | null = null; + private _searchInputElement: HTMLInputElement | null = null; private static readonly PAGINATION_SIZE = 'PaginationSize'; private readonly _subscriptions = new Subscription(); + private readonly _expandedRowKeys = new Set(); public get selectedItems(): CarbonListItem[] { const model = this._table.model; @@ -280,7 +304,8 @@ export class CarbonListComponent implements OnInit, AfterViewInit, OnDestroy { private readonly viewContentService: ViewContentService, private readonly keyStateService: KeyStateService, private readonly dragAndDropService: CarbonListDragAndDropService, - private readonly elementRef: ElementRef + private readonly elementRef: ElementRef, + private readonly cdr: ChangeDetectorRef ) { this.iconService.registerAll([ArrowDown16, ArrowUp16, SettingsView16, Draggable16]); } @@ -308,21 +333,9 @@ export class CarbonListComponent implements OnInit, AfterViewInit, OnDestroy { this._subscriptions.add( this.searchFormControl.valueChanges - .pipe(debounceTime(500)) + .pipe(debounceTime(this.searchDebounceMs)) .subscribe((searchString: string | null) => { - if (this.search.observed) { - this.search.emit(searchString); - return; - } - - if (!searchString) { - this._filteredItems$.next(null); - return; - } - - this._filteredItems$.next( - this.filterPipe.transform(this._completeDataSource, searchString ?? '') - ); + this.executeSearch(searchString); }) ); } @@ -463,51 +476,60 @@ export class CarbonListComponent implements OnInit, AfterViewInit, OnDestroy { ]).pipe( filter(([fields, items, viewInitialized]) => !!fields && !!items && viewInitialized), map(([fields, items]) => - items.map((item: CarbonListItem, index: number) => [ - ...this.getDragAndDropItemsItems(item, index, items.length), - ...fields.map((field: ColumnConfig) => { - switch (field.viewType) { - case ViewType.TEMPLATE: - return new TableItem({ - data: {item, index, length: items.length, ...field.templateData}, - item, - template: field.template, - }); - case ViewType.BOOLEAN: - let data = this.resolveObject(field, item); - data = !BOOLEAN_CONVERTER_VALUES.includes(data) - ? data - : `${'viewTypeConverter.' + data}`; - return new TableItem({ - data, - template: this.booleanTemplate, - item, - }); - case ViewType.TAGS: { - return new TableItem({ - data: { - tags: this.resolveTagObject(item, field.key), - tagAmount: field?.tagAmount || 1, - }, - item, - template: this.tagTemplate, - }); + items.map((item: CarbonListItem, index: number) => { + const row = [ + ...this.getDragAndDropItemsItems(item, index, items.length), + ...fields.map((field: ColumnConfig) => { + switch (field.viewType) { + case ViewType.TEMPLATE: + return new TableItem({ + data: {item, index, length: items.length, ...field.templateData}, + item, + template: field.template, + }); + case ViewType.BOOLEAN: + let data = this.resolveObject(field, item); + data = !BOOLEAN_CONVERTER_VALUES.includes(data) + ? data + : `${'viewTypeConverter.' + data}`; + return new TableItem({ + data, + template: this.booleanTemplate, + item, + }); + case ViewType.TAGS: { + return new TableItem({ + data: { + tags: this.resolveTagObject(item, field.key), + tagAmount: field?.tagAmount || 1, + }, + item, + template: this.tagTemplate, + }); + } + default: + const resolvedObject: string = this.resolveObject(field, item); + return new TableItem({ + title: resolvedObject ?? '-', + data: + (field.tooltipCharLimit + ? this.ellipsisPipe.transform(resolvedObject, field.tooltipCharLimit) + : resolvedObject) ?? '-', + template: this.defaultTemplate, + item, + }); } - default: - const resolvedObject: string = this.resolveObject(field, item); - return new TableItem({ - title: resolvedObject ?? '-', - data: - (field.tooltipCharLimit - ? this.ellipsisPipe.transform(resolvedObject, field.tooltipCharLimit) - : resolvedObject) ?? '-', - template: this.defaultTemplate, - item, - }); - } - }), - ...this.getExtraItems(item, index, items.length), - ]) + }), + ...this.getExtraItems(item, index, items.length), + ]; + + if (this.expandedRowTemplate && row.length > 0) { + row[0].expandedData = item; + row[0].expandedTemplate = this.expandedRowTemplate; + } + + return row; + }) ), tap((data: TableItem[][]) => { this._completeDataSource = data; @@ -521,10 +543,12 @@ export class CarbonListComponent implements OnInit, AfterViewInit, OnDestroy { this._tableItems$, this._filteredItems$, ]).pipe( + tap(() => this._captureExpandedRows()), map(([header, data, filteredData]) => { const model = new TableModel(); model.header = header; model.data = filteredData ?? data; + this._restoreExpandedRows(model); return model; }), startWith(new TableModel()) @@ -759,4 +783,282 @@ export class CarbonListComponent implements OnInit, AfterViewInit, OnDestroy { type: 'blue', })); } + + public getSearchSegments(): Array<{text: string; isInvalid: boolean}> { + const text = this.searchFormControl.value || ''; + if (!text) return [{text, isInvalid: false}]; + + const segments: Array<{text: string; isInvalid: boolean}> = []; + const fieldPattern = /(\w+(?:\.\w+)*):("([^"]+)"|(\S+))/g; + const invalidSet = new Set((this.invalidSearchFields || []).map(f => f.toLowerCase())); + + let lastIndex = 0; + let match: RegExpExecArray | null; + + while ((match = fieldPattern.exec(text)) !== null) { + if (match.index > lastIndex) { + segments.push({text: text.substring(lastIndex, match.index), isInvalid: false}); + } + + const fieldName = match[1]; + const isInvalid = invalidSet.has(fieldName.toLowerCase()); + segments.push({text: fieldName, isInvalid}); + + const rest = match[0].substring(fieldName.length); + segments.push({text: rest, isInvalid: false}); + + lastIndex = match.index + match[0].length; + } + + if (lastIndex < text.length) { + segments.push({text: text.substring(lastIndex), isInvalid: false}); + } + + return segments; + } + + private getSearchInputElement(): HTMLInputElement | null { + if (!this._searchInputElement) { + this._searchInputElement = this.elementRef.nativeElement.querySelector( + '.valtimo-search-container input' + ); + } + return this._searchInputElement; + } + + private getCurrentFieldToken(): {token: string; start: number} | null { + const value = this.searchFormControl.value || ''; + const input = this.getSearchInputElement(); + const cursor = input?.selectionStart ?? value.length; + + let start = value.lastIndexOf(' ', cursor - 1) + 1; + const beforeCursor = value.substring(start, cursor); + + if (beforeCursor.includes(':')) return null; + + const nextSpace = value.indexOf(' ', cursor); + const end = nextSpace === -1 ? value.length : nextSpace; + const afterCursor = value.substring(cursor, end); + + if (afterCursor.includes(':')) return null; + + return {token: beforeCursor, start}; + } + + public onSearchFocus(): void { + const input = this.getSearchInputElement(); + if (input && document.activeElement === input) { + this.updateAutocomplete(); + } + } + + public updateAutocomplete(): void { + const input = this.getSearchInputElement(); + + if (!input) { + this.showAutocomplete = false; + this.filteredSuggestions = []; + return; + } + + const tokenInfo = this.getCurrentFieldToken(); + + if (!tokenInfo || !this.searchFields?.length) { + this.showAutocomplete = false; + this.filteredSuggestions = []; + return; + } + + const searchToken = tokenInfo.token.toLowerCase(); + this.filteredSuggestions = searchToken.length === 0 + ? this.searchFields + : this.searchFields.filter( + field => + field.key.toLowerCase().includes(searchToken) || + (field.title && field.title.toLowerCase().includes(searchToken)) + ); + + this.showAutocomplete = this.filteredSuggestions.length > 0; + this.selectedSuggestionIndex = -1; + + if (this.showAutocomplete) { + this.autocompleteLeft = this.calculateTokenLeft(tokenInfo.start); + } + } + + private calculateTokenLeft(tokenStart: number): number { + const input = this.getSearchInputElement(); + if (!input) return 48; + + const value = this.searchFormControl.value || ''; + const textBefore = value.substring(0, tokenStart); + + const canvas = document.createElement('canvas'); + const ctx = canvas.getContext('2d'); + if (!ctx) return 48; + + const style = window.getComputedStyle(input); + ctx.font = `${style.fontSize} ${style.fontFamily}`; + const textWidth = ctx.measureText(textBefore).width; + + return 48 + textWidth - 12; + } + + public selectSuggestion(field: SearchField): void { + const tokenInfo = this.getCurrentFieldToken(); + if (!tokenInfo) return; + + const value = this.searchFormControl.value || ''; + const input = this.getSearchInputElement(); + const cursor = input?.selectionStart ?? value.length; + + const fieldPath = field.path?.replace(/^(doc|case):/, '') || field.key; + const newValue = + value.substring(0, tokenInfo.start) + fieldPath + ':' + value.substring(cursor); + + this.searchFormControl.setValue(newValue); + this.showAutocomplete = false; + + setTimeout(() => { + const newCursor = tokenInfo.start + fieldPath.length + 1; + input?.setSelectionRange(newCursor, newCursor); + input?.focus(); + }); + } + + public onSearchKeydown(event: KeyboardEvent): void { + if (event.key === 'Enter') { + event.preventDefault(); + this.onSearchEnter(); + return; + } + + if (!this.showAutocomplete || this.filteredSuggestions.length === 0) return; + + switch (event.key) { + case 'ArrowDown': + event.preventDefault(); + this.selectedSuggestionIndex = + this.selectedSuggestionIndex < 0 + ? 0 + : (this.selectedSuggestionIndex + 1) % this.filteredSuggestions.length; + break; + case 'ArrowUp': + event.preventDefault(); + this.selectedSuggestionIndex = + this.selectedSuggestionIndex < 0 + ? this.filteredSuggestions.length - 1 + : (this.selectedSuggestionIndex - 1 + this.filteredSuggestions.length) % + this.filteredSuggestions.length; + break; + case 'Tab': + event.preventDefault(); + if (this.selectedSuggestionIndex >= 0) { + this.selectSuggestion(this.filteredSuggestions[this.selectedSuggestionIndex]); + } else { + this.showAutocomplete = false; + } + break; + case 'Escape': + this.showAutocomplete = false; + break; + } + } + + public onSearchBlur(): void { + setTimeout(() => { + this.showAutocomplete = false; + this._searchInputElement = null; + }, 150); + } + + + public onSearchClear(): void { + this.showAutocomplete = false; + } + + private executeSearch(searchString: string | null): void { + if (searchString === this._lastExecutedSearch) { + return; + } + this._lastExecutedSearch = searchString; + + if (this.search.observed) { + this.search.emit(searchString); + return; + } + + if (!searchString) { + this._filteredItems$.next(null); + return; + } + + this._filteredItems$.next( + this.filterPipe.transform(this._completeDataSource, searchString ?? '') + ); + } + + public onSearchEnter(): void { + if (this.showAutocomplete && this.selectedSuggestionIndex >= 0) { + this.selectSuggestion(this.filteredSuggestions[this.selectedSuggestionIndex]); + } else { + this.showAutocomplete = false; + this.executeSearch(this.searchFormControl.value); + } + } + + private _searchOpen = false; + + public onSearchOpenChange(isOpen: boolean): void { + this._searchOpen = isOpen; + this._searchInputElement = null; + + if (isOpen) { + setTimeout(() => { + this.updateAutocomplete(); + }, 0); + } else { + this.showAutocomplete = false; + } + } + + public onSearchFocusOut(event: FocusEvent): void { + const container = this.elementRef.nativeElement.querySelector('.valtimo-search-container'); + const relatedTarget = event.relatedTarget as Node; + + if (!container?.contains(relatedTarget)) { + setTimeout(() => { + this.showAutocomplete = false; + this.cdr.markForCheck(); + }, 150); + } + } + + private _captureExpandedRows(): void { + if (!this.expandedRowKey || !this._table?.model) return; + + const model = this._table.model; + const items = this._items; + + for (let i = 0; i < items.length; i++) { + const key = _get(items[i], this.expandedRowKey); + if (key && model.isRowExpanded(i)) { + this._expandedRowKeys.add(key); + } else if (key) { + this._expandedRowKeys.delete(key); + } + } + } + + private _restoreExpandedRows(model: TableModel): void { + if (!this.expandedRowKey || this._expandedRowKeys.size === 0) return; + + const items = this._items; + for (let i = 0; i < items.length; i++) { + const key = _get(items[i], this.expandedRowKey); + if (key && this._expandedRowKeys.has(key)) { + model.expandRow(i, true); + } + } + } } diff --git a/frontend/projects/valtimo/components/src/lib/components/menu/services/menu.service.ts b/frontend/projects/valtimo/components/src/lib/components/menu/services/menu.service.ts index d6f7b86fe6..57b5acaaa6 100644 --- a/frontend/projects/valtimo/components/src/lib/components/menu/services/menu.service.ts +++ b/frontend/projects/valtimo/components/src/lib/components/menu/services/menu.service.ts @@ -1,5 +1,5 @@ /* - * Copyright 2015-2025 Ritense BV, the Netherlands. + * Copyright 2015-2026 Ritense BV, the Netherlands. * * Licensed under EUPL, Version 1.2 (the "License"); * you may not use this file except in compliance with the License. diff --git a/frontend/projects/valtimo/components/src/lib/models/index.ts b/frontend/projects/valtimo/components/src/lib/models/index.ts index 3199a424f7..60abde7744 100644 --- a/frontend/projects/valtimo/components/src/lib/models/index.ts +++ b/frontend/projects/valtimo/components/src/lib/models/index.ts @@ -1,5 +1,5 @@ /* - * Copyright 2015-2025 Ritense BV, the Netherlands. + * Copyright 2015-2026 Ritense BV, the Netherlands. * * Licensed under EUPL, Version 1.2 (the "License"); * you may not use this file except in compliance with the License. diff --git a/frontend/projects/valtimo/document/src/lib/models/advanced-document-search-request.ts b/frontend/projects/valtimo/document/src/lib/models/advanced-document-search-request.ts index 29bf206a7c..471b1ab297 100644 --- a/frontend/projects/valtimo/document/src/lib/models/advanced-document-search-request.ts +++ b/frontend/projects/valtimo/document/src/lib/models/advanced-document-search-request.ts @@ -37,6 +37,7 @@ export class AdvancedDocumentSearchRequestHttpBody { searchOperator?: SearchOperator; otherFilters?: Array; assigneeFilter?: AssigneeFilter; + globalSearchFilter?: string; } export class AdvancedDocumentSearchRequestImpl implements AdvancedDocumentSearchRequest { @@ -46,6 +47,7 @@ export class AdvancedDocumentSearchRequestImpl implements AdvancedDocumentSearch sort?: SortState; searchOperator?: SearchOperator; otherFilters?: Array; + globalSearchFilter?: string; constructor( definitionName: string, @@ -53,7 +55,8 @@ export class AdvancedDocumentSearchRequestImpl implements AdvancedDocumentSearch size: number, sort?: SortState, searchOperator?: SearchOperator, - otherFilters?: Array + otherFilters?: Array, + globalSearchFilter?: string ) { this.definitionName = definitionName; this.page = page; @@ -61,6 +64,7 @@ export class AdvancedDocumentSearchRequestImpl implements AdvancedDocumentSearch this.sort = sort; this.otherFilters = otherFilters; this.searchOperator = searchOperator; + this.globalSearchFilter = globalSearchFilter; } asHttpBody(): AdvancedDocumentSearchRequestHttpBody { @@ -74,6 +78,9 @@ export class AdvancedDocumentSearchRequestImpl implements AdvancedDocumentSearch if (this.searchOperator) { httpBody.searchOperator = this.searchOperator; } + if (this.globalSearchFilter) { + httpBody.globalSearchFilter = this.globalSearchFilter; + } return httpBody; } diff --git a/frontend/projects/valtimo/document/src/lib/services/document.service.ts b/frontend/projects/valtimo/document/src/lib/services/document.service.ts index f0835e071b..ce590676dc 100644 --- a/frontend/projects/valtimo/document/src/lib/services/document.service.ts +++ b/frontend/projects/valtimo/document/src/lib/services/document.service.ts @@ -1,5 +1,5 @@ /* - * Copyright 2015-2025 Ritense BV, the Netherlands. + * Copyright 2015-2026 Ritense BV, the Netherlands. * * Licensed under EUPL, Version 1.2 (the "License"); * you may not use this file except in compliance with the License. @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import {HttpClient, HttpHeaders, HttpParams} from '@angular/common/http'; +import {HttpClient, HttpErrorResponse, HttpHeaders, HttpParams} from '@angular/common/http'; import {Injectable} from '@angular/core'; import { AssigneeFilter, @@ -26,7 +26,7 @@ import { SearchOperator, TeamResponseDto, } from '@valtimo/shared'; -import {catchError, Observable, of, switchMap} from 'rxjs'; +import {BehaviorSubject, catchError, Observable, of, switchMap, tap} from 'rxjs'; import { AssignHandlerToDocumentResult, @@ -84,6 +84,9 @@ export class DocumentService { totalPages: 0, }; + private readonly _invalidSearchFields$ = new BehaviorSubject([]); + public readonly invalidSearchFields$ = this._invalidSearchFields$.asObservable(); + constructor( private http: HttpClient, private configService: ConfigService @@ -91,6 +94,23 @@ export class DocumentService { this.valtimoEndpointUri = this.configService.config.valtimoApi.endpointUri; } + public clearInvalidSearchFields(): void { + this._invalidSearchFields$.next([]); + } + + private extractInvalidSearchFields(error: HttpErrorResponse): string[] { + const message = error?.error?.detail || error?.error?.message || error?.error || ''; + const pluralMatch = message.match(/Unknown search field\(s\): (.+)/); + if (pluralMatch) { + return pluralMatch[1].split(', ').map((f: string) => f.trim()); + } + const singularMatch = message.match(/Unknown search field: (.+)/); + if (singularMatch) { + return [singularMatch[1].trim()]; + } + return []; + } + // Document-calls public getAllDefinitions(): Observable> { return this.http.get>( @@ -135,13 +155,22 @@ export class DocumentService { } public getDocuments(documentSearchRequest: DocumentSearchRequest): Observable { - return this.http.post( - `${this.valtimoEndpointUri}v1/document-search`, - documentSearchRequest.asHttpBody(), - { + return this.http + .post(`${this.valtimoEndpointUri}v1/document-search`, documentSearchRequest.asHttpBody(), { params: documentSearchRequest.asHttpParams(), - } - ); + headers: new HttpHeaders().set(InterceptorSkip, '500'), + }) + .pipe( + tap(() => this._invalidSearchFields$.next([])), + catchError((error: HttpErrorResponse) => { + const invalidFields = this.extractInvalidSearchFields(error); + if (invalidFields.length > 0) { + this._invalidSearchFields$.next(invalidFields); + return of(this.EMPTY_DOCUMENTS_RESPONSE as Documents); + } + throw error; + }) + ); } public getDocumentsSearch( @@ -150,7 +179,8 @@ export class DocumentService { assigneeFilter?: AssigneeFilter, otherFilters?: Array, statusFilter?: Array, - caseTagsFilter?: Array + caseTagsFilter?: Array, + globalSearchFilter?: string ): Observable { const body = { ...documentSearchRequest.asHttpBody(), @@ -159,15 +189,29 @@ export class DocumentService { ...(otherFilters && {otherFilters}), ...(statusFilter && {statusFilter}), ...(caseTagsFilter && {caseTagsFilter}), + ...(globalSearchFilter && {globalSearchFilter}), }; return this.http .post( `${this.valtimoEndpointUri}v1/document-definition/${documentSearchRequest.definitionName}/search`, body, - {params: documentSearchRequest.asHttpParams()} + { + params: documentSearchRequest.asHttpParams(), + headers: new HttpHeaders().set(InterceptorSkip, '500'), + } ) - .pipe(catchError(() => of(this.EMPTY_DOCUMENTS_RESPONSE as Documents))); + .pipe( + tap(() => this._invalidSearchFields$.next([])), + catchError((error: HttpErrorResponse) => { + const invalidFields = this.extractInvalidSearchFields(error); + if (invalidFields.length > 0) { + this._invalidSearchFields$.next(invalidFields); + return of(this.EMPTY_DOCUMENTS_RESPONSE as Documents); + } + throw error; + }) + ); } public getSpecifiedDocumentsSearch( @@ -176,7 +220,8 @@ export class DocumentService { assigneeFilter?: AssigneeFilter, otherFilters?: Array, statusFilter?: Array, - caseTagsFilter?: Array + caseTagsFilter?: Array, + globalSearchFilter?: string ): Observable { const body = { ...documentSearchRequest.asHttpBody(), @@ -185,15 +230,29 @@ export class DocumentService { ...(otherFilters && {otherFilters}), ...(statusFilter && {statusFilter}), ...(caseTagsFilter && {caseTagsFilter}), + ...(globalSearchFilter && {globalSearchFilter}), }; return this.http .post( `${this.valtimoEndpointUri}v1/case/${documentSearchRequest.definitionName}/search`, body, - {params: documentSearchRequest.asHttpParams()} + { + params: documentSearchRequest.asHttpParams(), + headers: new HttpHeaders().set(InterceptorSkip, '500'), + } ) - .pipe(catchError(() => of(this.EMPTY_DOCUMENTS_RESPONSE as SpecifiedDocuments))); + .pipe( + tap(() => this._invalidSearchFields$.next([])), + catchError((error: HttpErrorResponse) => { + const invalidFields = this.extractInvalidSearchFields(error); + if (invalidFields.length > 0) { + this._invalidSearchFields$.next(invalidFields); + return of(this.EMPTY_DOCUMENTS_RESPONSE as SpecifiedDocuments); + } + throw error; + }) + ); } public getDocumentSearchFields(caseDefinitionKey: string): Observable> { diff --git a/frontend/projects/valtimo/shared/assets/core/en.json b/frontend/projects/valtimo/shared/assets/core/en.json index e328368cfc..b283a6c85b 100644 --- a/frontend/projects/valtimo/shared/assets/core/en.json +++ b/frontend/projects/valtimo/shared/assets/core/en.json @@ -2143,7 +2143,9 @@ "showingResults": "Showing {{number}} of {{total}} results", "showingResult": "Showing {{number}} result", "automaticallyGenerated": "Automatically generated", - "search": "Search..." + "search": "Search...", + "globalSearchFilter": "Search entire document", + "globalSearchFilterPlaceholder": "Search across all document fields..." }, "webcam": {"takePicture": "Take picture", "save": "Save", "redo": "Redo"}, "customers": { @@ -3172,7 +3174,8 @@ "title": "Settings", "tabs": { "appearance": "Appearance", - "featureToggles": "Feature toggles" + "featureToggles": "Feature toggles", + "opensearch": "OpenSearch" }, "appearance": { "logo": { @@ -3208,6 +3211,13 @@ } }, "featureToggles": { + "searchEngine": { + "useOpenSearch": { + "title": "Use OpenSearch for case search", + "description": "When enabled, case searches use OpenSearch for better performance. When disabled, PostgreSQL is used.", + "unavailable": "OpenSearch is not configured for this application." + } + }, "refreshRequired": "Refresh required", "refreshModalText": "This setting requires a page refresh to take effect. Do you want to refresh now?", "refreshNow": "Refresh now", @@ -3299,6 +3309,64 @@ "menuCollapsedByDefault": { "title": "Menu collapsed by default", "description": "Start with the navigation menu collapsed instead of expanded." + }, + "enableGenericCaseList": { + "title": "Enable generic case list", + "description": "Show a single case list view with a dropdown to switch between case definitions." + } + } + }, + "opensearch": { + "title": "OpenSearch", + "reindex": { + "title": "Reindex Runs", + "description": "Reindex all documents from the database to OpenSearch. This operation runs in the background.", + "startButton": "Start Reindex", + "startModalTitle": "Start Reindex", + "detailTitle": "Run Details", + "parameters": "Parameters", + "results": "Results", + "reindexing": "Reindexing", + "pruning": "Pruning", + "counts": "Counts", + "status": "Status", + "processed": "Processed: {{current}} / {{total}}", + "skipped": "Skipped", + "pruned": "Pruned", + "started": "Started", + "elapsed": "Elapsed time", + "finished": "Finished", + "inProgress": "Reindexing in progress...", + "errorTitle": "Error", + "viewErrorLogs": "View error logs", + "pruneOrphans": "Prune orphaned documents", + "documentDefinitionName": "Document definition", + "documentDefinitionPlaceholder": "All document definitions", + "allDocumentDefinitions": "All document definitions", + "modifiedBefore": "Modified before", + "modifiedAfter": "Modified after", + "columns": { + "status": "Status", + "startedOn": "Started", + "finishedOn": "Finished", + "progress": "Progress", + "documentDefinition": "Document definition" + }, + "noResults": { + "title": "No reindex runs", + "description": "No reindex runs have been started yet." + }, + "statuses": { + "RUNNING": "Running", + "COMPLETED": "Completed", + "FAILED": "Failed", + "STOPPED": "Stopped", + "PRUNING": "Pruning orphaned documents" + }, + "pruningProgress": "Removed: {{removed}}", + "tooltips": { + "pruneOrphans": "Remove entries from OpenSearch that no longer exist in the database", + "pruned": "Search entries removed because the original document was deleted" } } } diff --git a/frontend/projects/valtimo/shared/assets/core/nl.json b/frontend/projects/valtimo/shared/assets/core/nl.json index d40dd4f0aa..4b4c6490b4 100644 --- a/frontend/projects/valtimo/shared/assets/core/nl.json +++ b/frontend/projects/valtimo/shared/assets/core/nl.json @@ -2170,7 +2170,9 @@ "showingResults": "{{number}} van {{total}} resultaten zichtbaar", "showingResult": "{{number}} resultaat zichtbaar", "automaticallyGenerated": "Automatisch gegenereerd", - "search": "Zoeken..." + "search": "Zoeken...", + "globalSearchFilter": "Zoek in geheel document", + "globalSearchFilterPlaceholder": "Zoek in alle documentvelden..." }, "webcam": {"takePicture": "Foto maken", "save": "Opslaan", "redo": "Opnieuw"}, "customers": { @@ -3205,7 +3207,8 @@ "title": "Instellingen", "tabs": { "appearance": "Weergave", - "featureToggles": "Functie-instellingen" + "featureToggles": "Functie-instellingen", + "opensearch": "OpenSearch" }, "appearance": { "logo": { @@ -3241,6 +3244,13 @@ } }, "featureToggles": { + "searchEngine": { + "useOpenSearch": { + "title": "Gebruik OpenSearch voor zaakzoekopdrachten", + "description": "Indien ingeschakeld worden zaakzoekopdrachten uitgevoerd met OpenSearch voor betere prestaties. Indien uitgeschakeld wordt PostgreSQL gebruikt.", + "unavailable": "OpenSearch is niet geconfigureerd voor deze applicatie." + } + }, "refreshRequired": "Vernieuwing vereist", "refreshModalText": "Deze instelling vereist een paginavernieuwing om effect te hebben. Wilt u nu vernieuwen?", "refreshNow": "Nu vernieuwen", @@ -3332,6 +3342,64 @@ "menuCollapsedByDefault": { "title": "Menu standaard ingeklapt", "description": "Start met het navigatiemenu ingeklapt in plaats van uitgeklapt." + }, + "enableGenericCaseList": { + "title": "Generieke zaaklijst inschakelen", + "description": "Toon één zaaklijstweergave met een dropdown om tussen zaakdefinities te wisselen." + } + } + }, + "opensearch": { + "title": "OpenSearch", + "reindex": { + "title": "Herindexering runs", + "description": "Herindexeer alle documenten van de database naar OpenSearch. Deze operatie draait op de achtergrond.", + "startButton": "Start herindexering", + "startModalTitle": "Start herindexering", + "detailTitle": "Run details", + "parameters": "Parameters", + "results": "Resultaten", + "reindexing": "Herindexering", + "pruning": "Opschonen", + "counts": "Aantallen", + "status": "Status", + "processed": "Verwerkt: {{current}} / {{total}}", + "skipped": "Overgeslagen", + "pruned": "Opgeschoond", + "started": "Gestart", + "elapsed": "Verstreken tijd", + "finished": "Voltooid", + "inProgress": "Herindexering bezig...", + "errorTitle": "Fout", + "viewErrorLogs": "Bekijk foutlogs", + "pruneOrphans": "Verwijder orphan documenten", + "documentDefinitionName": "Documentdefinitie", + "documentDefinitionPlaceholder": "Alle documentdefinities", + "allDocumentDefinitions": "Alle documentdefinities", + "modifiedBefore": "Gewijzigd voor", + "modifiedAfter": "Gewijzigd na", + "columns": { + "status": "Status", + "startedOn": "Gestart", + "finishedOn": "Voltooid", + "progress": "Voortgang", + "documentDefinition": "Documentdefinitie" + }, + "noResults": { + "title": "Geen herindexering runs", + "description": "Er zijn nog geen herindexering runs gestart." + }, + "statuses": { + "RUNNING": "Bezig", + "COMPLETED": "Voltooid", + "FAILED": "Mislukt", + "STOPPED": "Gestopt", + "PRUNING": "Orphan documenten opschonen" + }, + "pruningProgress": "Verwijderd: {{removed}}", + "tooltips": { + "pruneOrphans": "Verwijder documenten uit OpenSearch die niet meer in de database bestaan", + "pruned": "Documenten verwijderd uit OpenSearch omdat het oorspronkelijke document is verwijderd" } } } diff --git a/frontend/projects/valtimo/shared/src/lib/models/config.ts b/frontend/projects/valtimo/shared/src/lib/models/config.ts index 906df647d9..fa234a6fc7 100644 --- a/frontend/projects/valtimo/shared/src/lib/models/config.ts +++ b/frontend/projects/valtimo/shared/src/lib/models/config.ts @@ -1,17 +1,19 @@ /* - * Copyright 2015-2026 Ritense BV, the Netherlands. * - * Licensed under EUPL, Version 1.2 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * * Copyright 2015-2026 Ritense BV, the Netherlands. + * * + * * Licensed under EUPL, Version 1.2 (the "License"); + * * you may not use this file except in compliance with the License. + * * You may obtain a copy of the License at + * * + * * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" basis, + * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * * See the License for the specific language governing permissions and + * * limitations under the License. * - * https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" basis, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. */ import {InjectionToken, Injector} from '@angular/core'; @@ -94,6 +96,7 @@ interface ValtimoConfigFeatureToggles { enableIkoType?: boolean; enableGenericCaseList?: boolean; menuCollapsedByDefault?: boolean; + enableOpenSearch?: boolean; /** * @deprecated DMN decision table editing is always enabled and is no longer gated by a * feature toggle. This option is ignored and will be removed in a future major release. diff --git a/frontend/projects/valtimo/shared/src/lib/models/menu-item.model.ts b/frontend/projects/valtimo/shared/src/lib/models/menu-item.model.ts index c8425cd4b5..5228b48292 100644 --- a/frontend/projects/valtimo/shared/src/lib/models/menu-item.model.ts +++ b/frontend/projects/valtimo/shared/src/lib/models/menu-item.model.ts @@ -1,5 +1,6 @@ /* - * Copyright 2015-2025 Ritense BV, the Netherlands. + * + * Copyright 2015-2026 Ritense BV, the Netherlands. * * Licensed under EUPL, Version 1.2 (the "License"); * you may not use this file except in compliance with the License. @@ -12,12 +13,14 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. + * */ import {Observable} from 'rxjs'; enum IncludeFunction { ObjectManagementEnabled, + OpenSearchEnabled, ZgwFeaturesEnabled, } diff --git a/frontend/projects/valtimo/shared/src/lib/services/menu-include.service.ts b/frontend/projects/valtimo/shared/src/lib/services/menu-include.service.ts index 0221115bc4..8b5653bea7 100644 --- a/frontend/projects/valtimo/shared/src/lib/services/menu-include.service.ts +++ b/frontend/projects/valtimo/shared/src/lib/services/menu-include.service.ts @@ -1,5 +1,5 @@ /* - * Copyright 2015-2025 Ritense BV, the Netherlands. + * Copyright 2015-2026 Ritense BV, the Netherlands. * * Licensed under EUPL, Version 1.2 (the "License"); * you may not use this file except in compliance with the License. @@ -42,6 +42,8 @@ export class MenuIncludeService { switch (includeFunction) { case IncludeFunction.ObjectManagementEnabled: return this.configService.getFeatureToggleObservable('enableObjectManagement', true); + case IncludeFunction.OpenSearchEnabled: + return this.configService.getFeatureToggleObservable('enableOpenSearch', false); case IncludeFunction.ZgwFeaturesEnabled: return this.configService.getFeatureToggleObservable('enableZgwFeatures', false); default: diff --git a/gradle.properties b/gradle.properties index 7de5a54ebf..4827b98e65 100644 --- a/gradle.properties +++ b/gradle.properties @@ -35,6 +35,7 @@ operatonVersion=1.0.3 mybatisSpringBootStarterVersion=3.0.4 springBootVersion=3.5.16 +springDataOpenSearchVersion=1.6.1 springBootAdminStarterClientVersion=3.4.5 springDependencyManagementVersion=1.1.7 springCloudStreamVersion=4.2.1 diff --git a/settings.gradle b/settings.gradle index 86866528f9..6dbc48148f 100644 --- a/settings.gradle +++ b/settings.gradle @@ -40,6 +40,7 @@ include( ":backend:authorization", ":backend:building-block", ":backend:case", + ":backend:case-opensearch", ":backend:changelog", ":backend:command-handling", ":backend:contract",