Skip to content

feat: improve startup and save/read failure logging #47

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
anshuman9468 wants to merge 1 commit into
base: master
Choose a base branch
from
Open

feat: improve startup and save/read failure logging #47

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
45 changes: 38 additions & 7 deletions src/main/scala/org/dbpedia/databus/ApiImpl.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,13 +16,16 @@ import org.dbpedia.databus.swagger.model.{OperationFailure, OperationSuccess}
import sttp.model.Uri
import virtuoso.jdbc4.VirtuosoException

import org.slf4j.LoggerFactory
import scala.util.{Failure, Success, Try}
import scala.xml.Node
import collection.JavaConverters._


class ApiImpl(config: Config) extends DatabusApi {

private lazy val log = LoggerFactory.getLogger(this.getClass)

import ApiImpl._

private val client: GitClient = initGitClient(config)
Expand Down Expand Up @@ -122,6 +125,11 @@ class ApiImpl(config: Config) extends DatabusApi {
defaultLang,
lang))
.map(new String(_))
.recoverWith({
case e =>
log.error(s"Failed to read file $path for user $username: ${e.getMessage}")
Failure(e)
})
}

private def getLangFromAcceptHeader(request: HttpServletRequest) =
Expand Down Expand Up @@ -165,6 +173,11 @@ class ApiImpl(config: Config) extends DatabusApi {
} else {
Success(Unit)
}).flatMap(_ => client.commitSeveralFiles(username, fullFilenamesAndData))
.recoverWith({
case e =>
log.error(s"Failed to save files for user $username: ${e.getMessage}")
Failure(e)
})


private def deleteFileFromGit(username: String, path: String)(request: HttpServletRequest): Try[String] = {
Expand All @@ -177,16 +190,26 @@ class ApiImpl(config: Config) extends DatabusApi {

private def initGitClient(config: Config): GitClient = {
import config._
gitLocalDir.map(new LocalGitClient(_))
val client = gitLocalDir.map(new LocalGitClient(_))
.getOrElse({
log.info("Initializing remote Gitlab client")
val scheme = gitApiSchema.getOrElse("https")
val cl = for {
user <- gitApiUser
pass <- gitApiPass
host <- gitApiHost
} yield new RemoteGitlabHttpClient(user, pass, scheme, host, gitApiPort)
cl.getOrElse(throw new RuntimeException("Wrong remote git client configuration"))
cl.getOrElse({
val msg = "Wrong remote git client configuration: gitApiUser, gitApiPass, or gitApiHost is missing"
log.error(msg)
throw new RuntimeException(msg)
})
})

if (gitLocalDir.isDefined) {
log.info(s"Git client initialized with local directory: ${gitLocalDir.get}")
}
client
}

}
Expand Down Expand Up @@ -224,14 +247,22 @@ object ApiImpl {
private def fromMapper(mapper: Mapper): Config = {
implicit val mp = mapper

val defaultGraphIdPrefix = getParam("defaultGraphIdPrefix").get
def getRequiredParam(name: String): String = {
getParam(name).getOrElse({
val msg = s"Missing required configuration parameter: $name"
LoggerFactory.getLogger(ApiImpl.getClass).error(msg)
throw new NoSuchElementException(msg)
})
}

val defaultGraphIdPrefix = getRequiredParam("defaultGraphIdPrefix")

val storageSparqlEndpointUri = getParam("storageSparqlEndpointUri").get
val storageSparqlEndpointUri = getRequiredParam("storageSparqlEndpointUri")
val stUri = if (storageSparqlEndpointUri.endsWith("/")) storageSparqlEndpointUri.dropRight(1) else storageSparqlEndpointUri
val storageUser = getParam("storageUser").get
val storagePass = getParam("storagePass").get
val storageUser = getRequiredParam("storageUser")
val storagePass = getRequiredParam("storagePass")
val storageJdbcPort = getParam("storageJdbcPort").map(_.toInt)
val storageClass = getParam("storageClass").get
val storageClass = getRequiredParam("storageClass")
val storageDbName = getParam("storageDbName")

val gitLocalDir: Option[Path] = getParam("gitLocalDir").map(Paths.get(_))
Expand Down
17 changes: 15 additions & 2 deletions src/main/scala/org/dbpedia/databus/GitClient.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ import sttp.model.Uri
import org.json4s._
import org.json4s.JsonDSL._
import org.json4s.jackson.JsonMethods._
import org.slf4j.LoggerFactory

import scala.util.{Failure, Success, Try}

Expand All @@ -38,6 +39,8 @@ trait GitClient {

class LocalGitClient(rootPath: Path) extends GitClient {

private lazy val log = LoggerFactory.getLogger(this.getClass)

// here we do not cleanup, assuming that the number os repos is relatively low (less than 1 000 000)
private val locks = new ConcurrentHashMap[String, Object]()

Expand All @@ -62,6 +65,7 @@ class LocalGitClient(rootPath: Path) extends GitClient {
override def commitSeveralFiles(projectName: String, filenameAndData: Map[String, Array[Byte]]): Try[String] =
wrapWithSync(projectName) {
Try({
log.debug(s"Opening local git repository for project $projectName at ${getRepoPathFromUsername(projectName)}")
val git = Git.open(getRepoPathFromUsername(projectName).toFile)
val add = git.add()

Expand Down Expand Up @@ -139,16 +143,20 @@ class LocalGitClient(rootPath: Path) extends GitClient {

class RemoteGitlabHttpClient(rootUser: String, rootPass: String, scheme: String, hostname: String, port: Option[Int]) extends GitClient {

private lazy val log = LoggerFactory.getLogger(this.getClass)

private val baseUri = port
.map(p => Uri(scheme, hostname, p))
.getOrElse(Uri(scheme, hostname))
private val baseApiUri = baseUri.addPath("api", "v4")

private lazy val backend = HttpURLConnectionBackend()
private lazy val accessToken: Try[String] = Try {
log.info(s"Requesting access token from Gitlab at $baseUri")
val req = authReq(rootUser, rootPass)
backend.send(req).body match {
case Left(e) =>
log.error(s"Failed to get access token from Gitlab: $e")
Failure(new RuntimeException(e))
case Right(value) =>
val flds = for {
Expand Down Expand Up @@ -204,7 +212,9 @@ class RemoteGitlabHttpClient(rootUser: String, rootPass: String, scheme: String,
req.flatMap(r => {
val resp = r.send(backend)
resp.body match {
case Left(_) => Failure(new RuntimeException("Failed to get project id from gitlab"))
case Left(e) =>
log.error(s"Failed to get project id from gitlab for project $name: $e")
Failure(new RuntimeException("Failed to get project id from gitlab"))
case Right(li) =>
val ps = for {
JArray(a) <- parse(li)
Expand All @@ -221,7 +231,10 @@ class RemoteGitlabHttpClient(rootUser: String, rootPass: String, scheme: String,
// then just taking the first one is not the right approach
ids.headOption.map(i => i.toString)
.map(Success(_))
.getOrElse(Failure(new RuntimeException("Failed to get project id from gitlab")))
.getOrElse({
log.error(s"Project $name not found in gitlab")
Failure(new RuntimeException("Failed to get project id from gitlab"))
})
}
})

Expand Down