diff --git a/src/main/scala/org/dbpedia/databus/ApiImpl.scala b/src/main/scala/org/dbpedia/databus/ApiImpl.scala index e4df9ff..eeb152f 100644 --- a/src/main/scala/org/dbpedia/databus/ApiImpl.scala +++ b/src/main/scala/org/dbpedia/databus/ApiImpl.scala @@ -16,6 +16,7 @@ import org.dbpedia.databus.swagger.model.{OperationFailure, OperationSuccess} import sttp.model.Uri import virtuoso.jdbc4.VirtuosoException +import org.slf4j.LoggerFactory import scala.util.{Failure, Success, Try} import scala.xml.Node import collection.JavaConverters._ @@ -23,6 +24,8 @@ import collection.JavaConverters._ class ApiImpl(config: Config) extends DatabusApi { + private lazy val log = LoggerFactory.getLogger(this.getClass) + import ApiImpl._ private val client: GitClient = initGitClient(config) @@ -122,6 +125,11 @@ class ApiImpl(config: Config) extends DatabusApi { defaultLang, lang)) .map(new String(_)) + .recoverWith({ + case e => + log.error(s"Failed to read file $path for user $username: ${e.getMessage}") + Failure(e) + }) } private def getLangFromAcceptHeader(request: HttpServletRequest) = @@ -165,6 +173,11 @@ class ApiImpl(config: Config) extends DatabusApi { } else { Success(Unit) }).flatMap(_ => client.commitSeveralFiles(username, fullFilenamesAndData)) + .recoverWith({ + case e => + log.error(s"Failed to save files for user $username: ${e.getMessage}") + Failure(e) + }) private def deleteFileFromGit(username: String, path: String)(request: HttpServletRequest): Try[String] = { @@ -177,16 +190,26 @@ class ApiImpl(config: Config) extends DatabusApi { private def initGitClient(config: Config): GitClient = { import config._ - gitLocalDir.map(new LocalGitClient(_)) + val client = gitLocalDir.map(new LocalGitClient(_)) .getOrElse({ + log.info("Initializing remote Gitlab client") val scheme = gitApiSchema.getOrElse("https") val cl = for { user <- gitApiUser pass <- gitApiPass host <- gitApiHost } yield new RemoteGitlabHttpClient(user, pass, scheme, host, gitApiPort) - cl.getOrElse(throw new RuntimeException("Wrong remote git client configuration")) + cl.getOrElse({ + val msg = "Wrong remote git client configuration: gitApiUser, gitApiPass, or gitApiHost is missing" + log.error(msg) + throw new RuntimeException(msg) + }) }) + + if (gitLocalDir.isDefined) { + log.info(s"Git client initialized with local directory: ${gitLocalDir.get}") + } + client } } @@ -224,14 +247,22 @@ object ApiImpl { private def fromMapper(mapper: Mapper): Config = { implicit val mp = mapper - val defaultGraphIdPrefix = getParam("defaultGraphIdPrefix").get + def getRequiredParam(name: String): String = { + getParam(name).getOrElse({ + val msg = s"Missing required configuration parameter: $name" + LoggerFactory.getLogger(ApiImpl.getClass).error(msg) + throw new NoSuchElementException(msg) + }) + } + + val defaultGraphIdPrefix = getRequiredParam("defaultGraphIdPrefix") - val storageSparqlEndpointUri = getParam("storageSparqlEndpointUri").get + val storageSparqlEndpointUri = getRequiredParam("storageSparqlEndpointUri") val stUri = if (storageSparqlEndpointUri.endsWith("/")) storageSparqlEndpointUri.dropRight(1) else storageSparqlEndpointUri - val storageUser = getParam("storageUser").get - val storagePass = getParam("storagePass").get + val storageUser = getRequiredParam("storageUser") + val storagePass = getRequiredParam("storagePass") val storageJdbcPort = getParam("storageJdbcPort").map(_.toInt) - val storageClass = getParam("storageClass").get + val storageClass = getRequiredParam("storageClass") val storageDbName = getParam("storageDbName") val gitLocalDir: Option[Path] = getParam("gitLocalDir").map(Paths.get(_)) diff --git a/src/main/scala/org/dbpedia/databus/GitClient.scala b/src/main/scala/org/dbpedia/databus/GitClient.scala index 1db2bd4..30e57a6 100644 --- a/src/main/scala/org/dbpedia/databus/GitClient.scala +++ b/src/main/scala/org/dbpedia/databus/GitClient.scala @@ -13,6 +13,7 @@ import sttp.model.Uri import org.json4s._ import org.json4s.JsonDSL._ import org.json4s.jackson.JsonMethods._ +import org.slf4j.LoggerFactory import scala.util.{Failure, Success, Try} @@ -38,6 +39,8 @@ trait GitClient { class LocalGitClient(rootPath: Path) extends GitClient { + private lazy val log = LoggerFactory.getLogger(this.getClass) + // here we do not cleanup, assuming that the number os repos is relatively low (less than 1 000 000) private val locks = new ConcurrentHashMap[String, Object]() @@ -62,6 +65,7 @@ class LocalGitClient(rootPath: Path) extends GitClient { override def commitSeveralFiles(projectName: String, filenameAndData: Map[String, Array[Byte]]): Try[String] = wrapWithSync(projectName) { Try({ + log.debug(s"Opening local git repository for project $projectName at ${getRepoPathFromUsername(projectName)}") val git = Git.open(getRepoPathFromUsername(projectName).toFile) val add = git.add() @@ -139,6 +143,8 @@ class LocalGitClient(rootPath: Path) extends GitClient { class RemoteGitlabHttpClient(rootUser: String, rootPass: String, scheme: String, hostname: String, port: Option[Int]) extends GitClient { + private lazy val log = LoggerFactory.getLogger(this.getClass) + private val baseUri = port .map(p => Uri(scheme, hostname, p)) .getOrElse(Uri(scheme, hostname)) @@ -146,9 +152,11 @@ class RemoteGitlabHttpClient(rootUser: String, rootPass: String, scheme: String, private lazy val backend = HttpURLConnectionBackend() private lazy val accessToken: Try[String] = Try { + log.info(s"Requesting access token from Gitlab at $baseUri") val req = authReq(rootUser, rootPass) backend.send(req).body match { case Left(e) => + log.error(s"Failed to get access token from Gitlab: $e") Failure(new RuntimeException(e)) case Right(value) => val flds = for { @@ -204,7 +212,9 @@ class RemoteGitlabHttpClient(rootUser: String, rootPass: String, scheme: String, req.flatMap(r => { val resp = r.send(backend) resp.body match { - case Left(_) => Failure(new RuntimeException("Failed to get project id from gitlab")) + case Left(e) => + log.error(s"Failed to get project id from gitlab for project $name: $e") + Failure(new RuntimeException("Failed to get project id from gitlab")) case Right(li) => val ps = for { JArray(a) <- parse(li) @@ -221,7 +231,10 @@ class RemoteGitlabHttpClient(rootUser: String, rootPass: String, scheme: String, // then just taking the first one is not the right approach ids.headOption.map(i => i.toString) .map(Success(_)) - .getOrElse(Failure(new RuntimeException("Failed to get project id from gitlab"))) + .getOrElse({ + log.error(s"Project $name not found in gitlab") + Failure(new RuntimeException("Failed to get project id from gitlab")) + }) } })