Skip to content

Unify the main #415

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
slinkydeveloper merged 4 commits into from
May 15, 2026
Merged

Unify the main #415

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
def1c36
Unify the main
slinkydeveloper May 15, 2026
e7480d4
Exclude custom tests from the test suite if they're not configured
slinkydeveloper May 15, 2026
b247054
Spotless apply
slinkydeveloper May 15, 2026
ccb9f8e
More unification: all the kafka tests are now together, makes the con…
slinkydeveloper May 15, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
74 changes: 66 additions & 8 deletions e2e-tests/src/main/kotlin/dev/restate/sdktesting/junit/TestSuites.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,35 +8,93 @@
// https://github.com/restatedev/sdk-test-suite/blob/main/LICENSE
package dev.restate.sdktesting.junit

object TestSuites {
import dev.restate.sdktesting.tests.AwakeableIngressEndpointTest
import dev.restate.sdktesting.tests.AwakeableLeaderTransferTest
import dev.restate.sdktesting.tests.BackwardCompatibilityTest
import dev.restate.sdktesting.tests.ForwardCompatibilityTest
import dev.restate.sdktesting.tests.IngressTest
import dev.restate.sdktesting.tests.InvokerMemoryTest
import dev.restate.sdktesting.tests.JournalRetentionTest
import dev.restate.sdktesting.tests.KafkaTest
import dev.restate.sdktesting.tests.OpenAPITest
import dev.restate.sdktesting.tests.PauseResumeChangingDeploymentTest
import dev.restate.sdktesting.tests.PauseResumeTest
import dev.restate.sdktesting.tests.RestartAsNewInvocationTest
import dev.restate.sdktesting.tests.StatePatchingTest
import dev.restate.sdktesting.tests.TracingTest
import dev.restate.sdktesting.tests.UpgradeWithInFlightInvocation
import dev.restate.sdktesting.tests.UpgradeWithNewInvocation

object TestSuites : SuiteProvider {
override val defaultSuite: TestSuite
get() = DEFAULT_SUITE

val DEFAULT_SUITE =
TestSuite("default", emptyMap(), "none() | always-suspending | only-single-node")
TestSuite(
"default",
emptyMap(),
listOf(
clazz<AwakeableIngressEndpointTest>(),
clazz<IngressTest>(),
clazz<InvokerMemoryTest>(),
clazz<JournalRetentionTest>(),
clazz<KafkaTest>(),
clazz<OpenAPITest>(),
clazz<PauseResumeChangingDeploymentTest>(),
clazz<PauseResumeTest>(),
clazz<RestartAsNewInvocationTest>(),
clazz<StatePatchingTest>(),
clazz<TracingTest>(),
clazz<UpgradeWithNewInvocation>(),
clazz<UpgradeWithInFlightInvocation>(),
))

val THREE_NODES_SUITE =
TestSuite(
"threeNodes",
mapOf(
"RESTATE_DEFAULT_NUM_PARTITIONS" to "4",
),
"(none() | always-suspending | only-multi-node) & !only-single-node",
listOf(
clazz<AwakeableIngressEndpointTest>(),
clazz<AwakeableLeaderTransferTest>(),
clazz<IngressTest>(),
clazz<JournalRetentionTest>(),
clazz<PauseResumeTest>(),
clazz<RestartAsNewInvocationTest>(),
clazz<StatePatchingTest>(),
clazz<TracingTest>(),
),
3)

private val ALWAYS_SUSPENDING_SUITE =
TestSuite(
"alwaysSuspending",
mapOf("RESTATE_WORKER__INVOKER__INACTIVITY_TIMEOUT" to "0s"),
"always-suspending | only-always-suspending | only-single-node")
listOf(
clazz<InvokerMemoryTest>(),
clazz<PauseResumeChangingDeploymentTest>(),
clazz<UpgradeWithNewInvocation>(),
clazz<UpgradeWithInFlightInvocation>(),
))

private val THREE_NODES_ALWAYS_SUSPENDING_SUITE =
TestSuite(
"threeNodesAlwaysSuspending",
mapOf(
"RESTATE_WORKER__INVOKER__INACTIVITY_TIMEOUT" to "0s",
"RESTATE_DEFAULT_NUM_PARTITIONS" to "4",
),
"(always-suspending | only-always-suspending | only-multi-node) & !only-single-node",
listOf(clazz<AwakeableLeaderTransferTest>()),
3)

private val VERSION_COMPATIBILITY_SUITE =
TestSuite("versionCompat", emptyMap(), "version-compatibility")
TestSuite(
"versionCompat",
emptyMap(),
listOf(clazz<BackwardCompatibilityTest>(), clazz<ForwardCompatibilityTest>()))

fun allSuites(): List<TestSuite> {
override fun allSuites(): List<TestSuite> {
return listOf(
DEFAULT_SUITE,
THREE_NODES_SUITE,
Expand All @@ -45,7 +103,7 @@ object TestSuites {
VERSION_COMPATIBILITY_SUITE)
}

fun resolveSuites(suite: String?): List<TestSuite> {
override fun resolveSuites(suite: String?): List<TestSuite> {
return when (suite ?: "all") {
"all" -> allSuites()
else -> {
Expand Down
247 changes: 1 addition & 246 deletions e2e-tests/src/main/kotlin/dev/restate/sdktesting/main.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,251 +8,6 @@
// https://github.com/restatedev/sdk-test-suite/blob/main/LICENSE
package dev.restate.sdktesting

import com.charleskorn.kaml.Yaml
import com.charleskorn.kaml.decodeFromStream
import com.charleskorn.kaml.encodeToStream
import com.github.ajalt.clikt.core.CliktCommand
import com.github.ajalt.clikt.core.subcommands
import com.github.ajalt.clikt.parameters.arguments.*
import com.github.ajalt.clikt.parameters.groups.OptionGroup
import com.github.ajalt.clikt.parameters.groups.cooccurring
import com.github.ajalt.clikt.parameters.groups.provideDelegate
import com.github.ajalt.clikt.parameters.options.*
import com.github.ajalt.clikt.parameters.types.enum
import com.github.ajalt.clikt.parameters.types.path
import com.github.ajalt.mordant.rendering.TextColors.green
import com.github.ajalt.mordant.rendering.TextColors.red
import com.github.ajalt.mordant.rendering.TextStyles.bold
import com.github.ajalt.mordant.terminal.Terminal
import dev.restate.sdktesting.infra.*
import dev.restate.sdktesting.junit.ExecutionResult
import dev.restate.sdktesting.junit.TestSuites
import java.io.File
import java.io.FileInputStream
import java.io.FileOutputStream
import java.nio.file.Path
import java.time.LocalDateTime
import java.time.format.DateTimeFormatter
import kotlin.jvm.optionals.getOrNull
import kotlin.system.exitProcess
import kotlin.time.Duration
import kotlinx.serialization.Serializable
import org.junit.platform.engine.Filter
import org.junit.platform.engine.discovery.ClassNameFilter
import org.junit.platform.engine.support.descriptor.MethodSource
import org.junit.platform.launcher.MethodFilter

@Serializable data class ExclusionsFile(val exclusions: Map<String, List<String>>? = emptyMap())

class RestateSdkTestSuite : CliktCommand() {
override fun run() {
// Disable log4j2 JMX, this prevents reconfiguration
System.setProperty("log4j2.disable.jmx", "true")
// This is hours of debugging, don't touch it
// tl;dr this makes sure a single log4j2 configuration exists for the whole JVM,
// important to make Configurator.reconfigure work
System.setProperty(
"log4j2.contextSelector", "org.apache.logging.log4j.core.selector.BasicContextSelector")
// The default keep alive time is way too long, and this is a problem when we stop and restart
// containers.
System.setProperty("jdk.httpclient.keepalive.timeout", "5")
// The health check strategy uses the HttpUrlConnection which has no connect timeout by default.
// Could have caused the health check to hang indefinitely.
System.setProperty("sun.net.client.defaultConnectTimeout", "5000")
// Enable Logging of JDK client
// System.setProperty("java.util.logging.manager", "org.apache.logging.log4j.jul.LogManager")
// System.setProperty("jdk.httpclient.HttpClient.log", "all")
}
}

class TestRunnerOptions : OptionGroup() {
val restateContainerImage by
option(envvar = "RESTATE_CONTAINER_IMAGE").help("Image used to run Restate")
val reportDir by
option(envvar = "TEST_REPORT_DIR").path().help("Base report directory").defaultLazy {
defaultReportDirectory()
}
val imagePullPolicy by
option()
.enum<PullPolicy>()
.help(
"Pull policy used to pull containers required for testing. In case of ALWAYS, docker won't pull images with repository prefix restate.local or localhost")
.default(PullPolicy.ALWAYS)

fun applyToDeployerConfig(deployerConfig: RestateDeployerConfig): RestateDeployerConfig {
var newConfig = deployerConfig
if (restateContainerImage != null) {
newConfig = newConfig.copy(restateContainerImage = restateContainerImage!!)
}
newConfig = newConfig.copy(imagePullPolicy = imagePullPolicy)
return newConfig
}

private fun defaultReportDirectory(): Path {
val formatter = DateTimeFormatter.ofPattern("yyyyMMdd_HHmmss")
return Path.of("test_report/${LocalDateTime.now().format(formatter)}").toAbsolutePath()
}
}

class FilterOptions : OptionGroup() {
val testSuite by
option()
.required()
.help(
"Test suite to run. Available: ${listOf("all") + TestSuites.allSuites().map { it.name }}")
val testName by option().help("Name of the test to run for the given suite")
}

abstract class TestRunCommand(help: String) : CliktCommand(help) {
val testRunnerOptions by TestRunnerOptions()
}

class Run :
TestRunCommand(
"""
Run test suite, executing the service as container.
"""
.trimIndent()) {
val filter by FilterOptions().cooccurring()
val exclusionsFile by
option("--exclusions", "--exclusions-file", envvar = "TEST_EXCLUSIONS_FILE")
.help("File containing the excluded tests")
val parallel by
option(help = "Enable parallel testing")
.help(
"If set, runs tests in parallel. We suggest running tests sequentially when using podman")
.flag("--sequential", default = true)

override fun run() {
val terminal = Terminal()

val restateDeployerConfig =
RestateDeployerConfig(
mapOf(),
)

// Register global config of the deployer
registerGlobalConfig(testRunnerOptions.applyToDeployerConfig(restateDeployerConfig))

// Resolve test configurations
val testSuites = TestSuites.resolveSuites(filter?.testSuite)

// Load exclusions file
val loadedExclusions: ExclusionsFile =
if (exclusionsFile != null) {
FileInputStream(File(exclusionsFile!!))
.use { Yaml.default.decodeFromStream<ExclusionsFile>(it) }
.also { exclusions ->
println("Using exclusions file $exclusionsFile")
println("Loaded exclusions: ${exclusions.exclusions}")
}
} else {
ExclusionsFile()
}

val reports = mutableListOf<ExecutionResult>()
val newExclusions = mutableMapOf<String, List<String>>()
var newFailures = false
for (testSuite in testSuites) {
val exclusions = loadedExclusions.exclusions?.get(testSuite.name) ?: emptyList()
val exclusionsFilters =
if (exclusions.isNotEmpty()) {
listOf(MethodFilter.excludeMethodNamePatterns(exclusions))
} else {
listOf()
}

val cliOptionFilter =
filter?.testName?.let {
listOf(ClassNameFilter.includeClassNamePatterns(testClassNameToFQCN(it)))
} ?: emptyList<Filter<*>>()

val report =
testSuite.runTests(
terminal,
testRunnerOptions.reportDir,
exclusionsFilters + cliOptionFilter,
false,
parallel)

reports.add(report)
// No need to wait the end of the run for this
report.printFailuresToFiles(testRunnerOptions.reportDir)
val failures = report.failedTests
if (failures.isNotEmpty() || exclusions.isNotEmpty()) {
newExclusions[testSuite.name] =
(failures
.mapNotNull { it.source.getOrNull() }
.mapNotNull {
when (it) {
is MethodSource -> "${it.className}.${it.methodName}"
else -> null
}
}
.distinct() + exclusions)
.sorted()
}
if (failures.isNotEmpty()) {
newFailures = true
}
}

// Write out the exclusions file
FileOutputStream(testRunnerOptions.reportDir.resolve("exclusions.new.yaml").toFile()).use {
Yaml.default.encodeToStream(ExclusionsFile(newExclusions.toSortedMap()), it)
}

// Print final report
val succeededTests = reports.sumOf { it.succeededTests }
val executedTests = reports.sumOf { it.executedTests }
val testsStyle = if (succeededTests == executedTests) green else red
val testsInfoLine = testsStyle("""* Succeeded tests: $succeededTests / ${executedTests}""")

val failedClasses = reports.sumOf { it.executedClasses - it.succeededClasses }
val classesStyle = if (failedClasses != 0) red else green
val classesInfoLine = classesStyle("""* Failed classes initialization: $failedClasses""")

val totalDuration = reports.fold(Duration.ZERO) { d, res -> d + res.executionDuration }

println(
"""
${bold("========================= Final results =========================")}
🗈 Report directory: ${testRunnerOptions.reportDir}
* Run test suites: ${reports.map { it.testSuite }}
$testsInfoLine
$classesInfoLine
* Execution time: $totalDuration
"""
.trimIndent())

for (report in reports) {
report.printFailuresToTerminal(terminal)
}

exitProcess(
if (newFailures) {
1
} else {
0
})
}
}

fun main(args: Array<String>) {
val args =
if (args.isEmpty()) {
arrayOf("run")
} else {
args
}

RestateSdkTestSuite().subcommands(Run()).main(args)
}

private fun testClassNameToFQCN(className: String): String {
if (className.contains('.')) {
// Then it's FQCN
return className
}
return "dev.restate.sdktesting.tests.${className}"
}
fun main(args: Array<String>) = runMain(args, TestSuites)
2 changes: 0 additions & 2 deletions e2e-tests/src/main/kotlin/dev/restate/sdktesting/tests/AwakeableLeaderTransferTest.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,6 @@ import org.apache.logging.log4j.LogManager
import org.assertj.core.api.Assertions.assertThat
import org.awaitility.kotlin.await
import org.awaitility.kotlin.withAlias
import org.junit.jupiter.api.Tag
import org.junit.jupiter.api.Test
import org.junit.jupiter.api.Timeout
import org.junit.jupiter.api.extension.RegisterExtension
Expand Down Expand Up @@ -76,7 +75,6 @@ class AwakeableLeaderTransferTest {

@Test
@Timeout(180)
@Tag("only-multi-node")
fun awakeableCompletionsAreNotLostDuringLeaderTransfer(
@InjectClient ingressClient: Client,
@InjectContainerHandle(hostName = RESTATE_RUNTIME) runtimeHandle: ContainerHandle,
Expand Down
Loading
Loading