diff --git a/golem-xiv-ddgs-client/build.gradle.kts b/golem-xiv-ddgs-client/build.gradle.kts new file mode 100644 index 0000000..33ba0ac --- /dev/null +++ b/golem-xiv-ddgs-client/build.gradle.kts @@ -0,0 +1,39 @@ +/* + * Golem XIV - Autonomous metacognitive AI system with semantic memory and self-directed research + * Copyright (C) 2025 Kazimierz Pogoda / Xemantic + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +plugins { + alias(libs.plugins.kotlin.jvm) + alias(libs.plugins.kotlin.plugin.serialization) + id("golem.convention") +} + +dependencies { + api(libs.ktor.client.core) + api(libs.ktor.client.java) + api(libs.ktor.client.content.negotiation) + api(libs.ktor.serialization.kotlinx.json) + api(libs.kotlinx.serialization.json) + + implementation(project(":golem-xiv-logging")) + implementation(libs.logback.classic) + + testImplementation(libs.kotlin.test) + testImplementation(libs.kotlinx.coroutines.test) + testImplementation(libs.xemantic.kotlin.test) + testImplementation(libs.testcontainers) +} diff --git a/golem-xiv-ddgs-client/src/main/kotlin/com/xemantic/ai/golem/ddgs/DdgsClient.kt b/golem-xiv-ddgs-client/src/main/kotlin/com/xemantic/ai/golem/ddgs/DdgsClient.kt new file mode 100644 index 0000000..537601a --- /dev/null +++ b/golem-xiv-ddgs-client/src/main/kotlin/com/xemantic/ai/golem/ddgs/DdgsClient.kt @@ -0,0 +1,225 @@ +/* + * Golem XIV - Autonomous metacognitive AI system with semantic memory and self-directed research + * Copyright (C) 2025 Kazimierz Pogoda / Xemantic + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.xemantic.ai.golem.ddgs + +import io.github.oshai.kotlinlogging.KotlinLogging +import io.ktor.client.* +import io.ktor.client.call.* +import io.ktor.client.plugins.* +import io.ktor.client.plugins.contentnegotiation.* +import io.ktor.client.request.* +import io.ktor.http.* +import io.ktor.serialization.kotlinx.json.* +import kotlinx.serialization.json.Json + +/** + * Client for DDGS (DuckDuckGo Search) API server. + * + * @param baseUrl The base URL of the DDGS server (default: http://localhost:8000) + */ +class DdgsClient( + private val baseUrl: String = "http://localhost:8000" +) : AutoCloseable { + + private val logger = KotlinLogging.logger {} + + private val httpClient = HttpClient { + install(ContentNegotiation) { + json(Json { + ignoreUnknownKeys = true + prettyPrint = true + }) + } + install(HttpTimeout) { + requestTimeoutMillis = 30_000 + connectTimeoutMillis = 10_000 + } + install(HttpRequestRetry) { + retryOnServerErrors(maxRetries = 3) + exponentialDelay() + } + defaultRequest { + url(baseUrl) + } + } + + /** + * Search for text results. + * + * @param query The search query + * @param region The region to search in (e.g., "wt-wt" for worldwide) + * @param safesearch Safe search level: "on", "moderate", or "off" + * @param timelimit Time limit for results (e.g., "d" for day, "w" for week, "m" for month) + * @param maxResults Maximum number of results to return + * @param backend Search backend to use (e.g., "duckduckgo", "bing", "google") + * @return List of text search results + */ + suspend fun searchText( + query: String, + region: String? = null, + safesearch: String? = null, + timelimit: String? = null, + maxResults: Int? = null, + backend: String? = null + ): List { + logger.debug { "Searching text: query='$query', region=$region, safesearch=$safesearch" } + + return httpClient.post("/search/text") { + contentType(ContentType.Application.Json) + setBody(TextSearchRequest( + query = query, + region = region, + safesearch = safesearch, + timelimit = timelimit, + maxResults = maxResults, + backend = backend + )) + }.body>().results + } + + /** + * Search for images. + * + * @param query The search query + * @param region The region to search in + * @param safesearch Safe search level + * @param timelimit Time limit for results + * @param maxResults Maximum number of results to return + * @return List of image search results + */ + suspend fun searchImages( + query: String, + region: String? = null, + safesearch: String? = null, + timelimit: String? = null, + maxResults: Int? = null + ): List { + logger.debug { "Searching images: query='$query'" } + + return httpClient.post("/search/images") { + contentType(ContentType.Application.Json) + setBody(ImageSearchRequest( + query = query, + region = region, + safesearch = safesearch, + timelimit = timelimit, + maxResults = maxResults + )) + }.body>().results + } + + /** + * Search for news. + * + * @param query The search query + * @param region The region to search in + * @param safesearch Safe search level + * @param timelimit Time limit for results + * @param maxResults Maximum number of results to return + * @return List of news search results + */ + suspend fun searchNews( + query: String, + region: String? = null, + safesearch: String? = null, + timelimit: String? = null, + maxResults: Int? = null + ): List { + logger.debug { "Searching news: query='$query'" } + + return httpClient.post("/search/news") { + contentType(ContentType.Application.Json) + setBody(NewsSearchRequest( + query = query, + region = region, + safesearch = safesearch, + timelimit = timelimit, + maxResults = maxResults + )) + }.body>().results + } + + /** + * Search for videos. + * + * @param query The search query + * @param region The region to search in + * @param safesearch Safe search level + * @param timelimit Time limit for results + * @param maxResults Maximum number of results to return + * @return List of video search results + */ + suspend fun searchVideos( + query: String, + region: String? = null, + safesearch: String? = null, + timelimit: String? = null, + maxResults: Int? = null + ): List { + logger.debug { "Searching videos: query='$query'" } + + return httpClient.post("/search/videos") { + contentType(ContentType.Application.Json) + setBody(VideoSearchRequest( + query = query, + region = region, + safesearch = safesearch, + timelimit = timelimit, + maxResults = maxResults + )) + }.body>().results + } + + /** + * Search for books. + * + * @param query The search query + * @param maxResults Maximum number of results to return + * @return List of book search results + */ + suspend fun searchBooks( + query: String, + maxResults: Int? = null + ): List { + logger.debug { "Searching books: query='$query'" } + + return httpClient.post("/search/books") { + contentType(ContentType.Application.Json) + setBody(BookSearchRequest( + query = query, + maxResults = maxResults + )) + }.body>().results + } + + /** + * Check the health status of the DDGS server. + * + * @return Health status response + */ + suspend fun checkHealth(): HealthStatus { + logger.debug { "Checking DDGS server health" } + return httpClient.get("/health").body() + } + + override fun close() { + logger.debug { "Closing DDGS client" } + httpClient.close() + } +} diff --git a/golem-xiv-ddgs-client/src/main/kotlin/com/xemantic/ai/golem/ddgs/DdgsModel.kt b/golem-xiv-ddgs-client/src/main/kotlin/com/xemantic/ai/golem/ddgs/DdgsModel.kt new file mode 100644 index 0000000..21bb626 --- /dev/null +++ b/golem-xiv-ddgs-client/src/main/kotlin/com/xemantic/ai/golem/ddgs/DdgsModel.kt @@ -0,0 +1,141 @@ +/* + * Golem XIV - Autonomous metacognitive AI system with semantic memory and self-directed research + * Copyright (C) 2025 Kazimierz Pogoda / Xemantic + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.xemantic.ai.golem.ddgs + +import kotlinx.serialization.SerialName +import kotlinx.serialization.Serializable + +// Request DTOs + +@Serializable +data class TextSearchRequest( + val query: String, + val region: String? = null, + val safesearch: String? = null, + val timelimit: String? = null, + @SerialName("max_results") + val maxResults: Int? = null, + val backend: String? = null +) + +@Serializable +data class ImageSearchRequest( + val query: String, + val region: String? = null, + val safesearch: String? = null, + val timelimit: String? = null, + @SerialName("max_results") + val maxResults: Int? = null +) + +@Serializable +data class NewsSearchRequest( + val query: String, + val region: String? = null, + val safesearch: String? = null, + val timelimit: String? = null, + @SerialName("max_results") + val maxResults: Int? = null +) + +@Serializable +data class VideoSearchRequest( + val query: String, + val region: String? = null, + val safesearch: String? = null, + val timelimit: String? = null, + @SerialName("max_results") + val maxResults: Int? = null +) + +@Serializable +data class BookSearchRequest( + val query: String, + @SerialName("max_results") + val maxResults: Int? = null +) + +// Response DTOs + +@Serializable +data class SearchResponse( + val results: List +) + +@Serializable +data class TextSearchResult( + val title: String, + val href: String, + val body: String +) + +@Serializable +data class ImageSearchResult( + val title: String, + val image: String, + val thumbnail: String, + val url: String, + val height: Int, + val width: Int, + val source: String +) + +@Serializable +data class NewsSearchResult( + val date: String, + val title: String, + val body: String, + val url: String, + val image: String? = null, + val source: String +) + +@Serializable +data class VideoImages( + val large: String, + val medium: String, + val motion: String, + val small: String +) + +@Serializable +data class VideoSearchResult( + val title: String, + val description: String, + val duration: String, + @SerialName("embed_url") + val embedUrl: String, + val images: VideoImages, + val uploader: String +) + +@Serializable +data class BookSearchResult( + val title: String, + val author: String, + val publisher: String, + val info: String, + val url: String, + val thumbnail: String +) + +@Serializable +data class HealthStatus( + val status: String +) diff --git a/golem-xiv-ddgs-client/src/test/kotlin/com/xemantic/ai/golem/ddgs/DdgsClientTest.kt b/golem-xiv-ddgs-client/src/test/kotlin/com/xemantic/ai/golem/ddgs/DdgsClientTest.kt new file mode 100644 index 0000000..90ecfec --- /dev/null +++ b/golem-xiv-ddgs-client/src/test/kotlin/com/xemantic/ai/golem/ddgs/DdgsClientTest.kt @@ -0,0 +1,150 @@ +/* + * Golem XIV - Autonomous metacognitive AI system with semantic memory and self-directed research + * Copyright (C) 2025 Kazimierz Pogoda / Xemantic + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.xemantic.ai.golem.ddgs + +import com.xemantic.kotlin.test.have +import com.xemantic.kotlin.test.should +import kotlinx.coroutines.test.runTest +import org.junit.jupiter.api.AfterAll +import org.junit.jupiter.api.Test + +class DdgsClientTest { + + companion object { + @JvmStatic + @AfterAll + fun tearDown() { + TestDdgs.stop() + } + } + + @Test + fun `should check health status`() = runTest { + // given + val client = DdgsClient(baseUrl = TestDdgs.baseUrl) + + // when + val health = client.checkHealth() + + // then + health should { + have(status == "healthy") + } + + client.close() + } + + @Test + fun `should search text`() = runTest { + // given + val client = DdgsClient(baseUrl = TestDdgs.baseUrl) + + // when + val results = client.searchText( + query = "Kotlin programming language", + maxResults = 5 + ) + + // then + assert(results.isNotEmpty()) + assert(results.all { it.title.isNotBlank() }) + assert(results.all { it.href.isNotBlank() }) + assert(results.all { it.body.isNotBlank() }) + + client.close() + } + + @Test + fun `should search images`() = runTest { + // given + val client = DdgsClient(baseUrl = TestDdgs.baseUrl) + + // when + val results = client.searchImages( + query = "kotlin logo", + maxResults = 3 + ) + + // then + assert(results.isNotEmpty()) + assert(results.all { it.title.isNotBlank() }) + assert(results.all { it.image.isNotBlank() }) + assert(results.all { it.url.isNotBlank() }) + + client.close() + } + + @Test + fun `should search news`() = runTest { + // given + val client = DdgsClient(baseUrl = TestDdgs.baseUrl) + + // when + val results = client.searchNews( + query = "technology", + maxResults = 5 + ) + + // then + assert(results.isNotEmpty()) + assert(results.all { it.title.isNotBlank() }) + assert(results.all { it.url.isNotBlank() }) + assert(results.all { it.body.isNotBlank() }) + + client.close() + } + + @Test + fun `should search videos`() = runTest { + // given + val client = DdgsClient(baseUrl = TestDdgs.baseUrl) + + // when + val results = client.searchVideos( + query = "kotlin tutorial", + maxResults = 3 + ) + + // then + assert(results.isNotEmpty()) + assert(results.all { it.title.isNotBlank() }) + assert(results.all { it.embedUrl.isNotBlank() }) + + client.close() + } + + @Test + fun `should search books`() = runTest { + // given + val client = DdgsClient(baseUrl = TestDdgs.baseUrl) + + // when + val results = client.searchBooks( + query = "programming", + maxResults = 5 + ) + + // then + assert(results.isNotEmpty()) + assert(results.all { it.title.isNotBlank() }) + assert(results.all { it.url.isNotBlank() }) + + client.close() + } +} diff --git a/golem-xiv-ddgs-client/src/test/kotlin/com/xemantic/ai/golem/ddgs/TestDdgs.kt b/golem-xiv-ddgs-client/src/test/kotlin/com/xemantic/ai/golem/ddgs/TestDdgs.kt new file mode 100644 index 0000000..fdeef8d --- /dev/null +++ b/golem-xiv-ddgs-client/src/test/kotlin/com/xemantic/ai/golem/ddgs/TestDdgs.kt @@ -0,0 +1,56 @@ +/* + * Golem XIV - Autonomous metacognitive AI system with semantic memory and self-directed research + * Copyright (C) 2025 Kazimierz Pogoda / Xemantic + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +package com.xemantic.ai.golem.ddgs + +import org.testcontainers.containers.GenericContainer +import org.testcontainers.containers.wait.strategy.Wait +import org.testcontainers.utility.DockerImageName + +/** + * Test utility for managing DDGS Docker container in integration tests. + */ +object TestDdgs { + + private const val DDGS_PORT = 8000 + + /** + * Docker image to use for tests. + * Uses local image when available, falls back to GitHub Container Registry image in CI. + */ + private val dockerImage: String = System.getenv("CI")?.let { + "ghcr.io/xemantic/ddgs:latest" + } ?: "ddgs:latest" + + private val container: GenericContainer<*> by lazy { + GenericContainer(DockerImageName.parse(dockerImage)) + .withExposedPorts(DDGS_PORT) + .waitingFor(Wait.forHttp("/health").forStatusCode(200)) + .apply { + start() + } + } + + val baseUrl: String by lazy { + "http://${container.host}:${container.getMappedPort(DDGS_PORT)}" + } + + fun stop() { + container.stop() + } +} diff --git a/golem-xiv-ddgs-client/src/test/resources/logback.xml b/golem-xiv-ddgs-client/src/test/resources/logback.xml new file mode 100644 index 0000000..1f35dcc --- /dev/null +++ b/golem-xiv-ddgs-client/src/test/resources/logback.xml @@ -0,0 +1,42 @@ + + + + + + + + + + true + + + + + + + + %d{HH:mm:ss.SSS} %highlight(%-5level) %cyan([%thread]) %magenta(%logger{36}) - %msg%n + + + + + + + + diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 5d6fb4e..7c21ef0 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -23,6 +23,7 @@ neo4jBrowser = "2025.8.0" neo4jBrowserSha256 = "4ed8ca29a27519a79f7013258acbcb020649a7fef354e1e157c1dc8f0a83cef2" neo4jDriver = "6.0.2" neo4jMigrations = "3.1.0" +testcontainers = "2.0.3" slf4j = "2.0.17" logback = "1.5.23" @@ -71,6 +72,8 @@ neo4j-harness = { module = "org.neo4j.test:neo4j-harness", version.ref = "neo4j" neo4j-java-driver = { module = "org.neo4j.driver:neo4j-java-driver", version.ref = "neo4jDriver" } neo4j-migrations = { module = "eu.michael-simons.neo4j:neo4j-migrations", version.ref = "neo4jMigrations" } +testcontainers = { module = "org.testcontainers:testcontainers", version.ref = "testcontainers" } + ktor-sse = { module = "io.ktor:ktor-sse", version.ref = "ktor" } ktor-client-core = { module = "io.ktor:ktor-client-core", version.ref = "ktor" } ktor-client-java = { module = "io.ktor:ktor-client-java", version.ref = "ktor" } diff --git a/settings.gradle.kts b/settings.gradle.kts index 157b9b9..4962ea2 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -34,6 +34,7 @@ include( ":golem-xiv-cognizer-anthropic", // ":golem-xiv-cognizer-dashscope", ":golem-xiv-playwright", + ":golem-xiv-ddgs-client", ":golem-xiv-server", ":golem-xiv-presenter", ":golem-xiv-web",