[Bounty] Headless browser module #40

headless/Dockerfile

@@ -0,0 +1,50 @@
+FROM fedora:latest AS builder
+
+ARG TARGETARCH
+
+RUN yum install gcc glibc-devel zlib-devel libstdc++-static curl git -y
+RUN yum install zlib-devel zlib-static -y
+RUN yum install glibc-static -y
+
+# Install Enterprise GraalVM and Dependencies (needed to compile native image with JNI invocations)
+RUN cd / && mkdir graal
+
+COPY resources/graalvm-ee-java19-${TARGETARCH}.tar.gz /graal/graal.tar
+COPY resources/graal-config /graal/config
+
+WORKDIR /graal
+
+RUN tar -xzf graal.tar && rm graal.tar
+
+ENV PATH="${PATH}:/graal/graalvm-ee-java19-22.3.0/bin"
+ENV JAVA_HOME="/graal/graalvm-ee-java19-22.3.0"
+
+RUN gu install native-image -A --config /graal/config
+
+COPY resources/pod-jaydeesimon-jsoup-0.1-standalone.jar jsoup-pod.jar
+RUN native-image -jar jsoup-pod.jar -H:Name=finder --static -H:+ReportExceptionStackTraces -J-Dclojure.spec.skip-macros=true -J-Dclojure.compiler.direct-linking=true -H:Log=registerResource: --initialize-at-build-time -H:EnableURLProtocols=http,https --verbose --allow-incomplete-classpath --no-fallback --no-server -J-Xmx8g
+
+FROM debian:stretch AS runner
+USER root
+
+# Install chromedriver
+RUN apt-get update && apt install chromium-driver curl -y
+
+# Install script interpreter
+RUN curl -sLO https://raw.githubusercontent.com/babashka/babashka/master/install
+RUN chmod a+x install
+RUN ["/bin/bash", "./install", "--static"]
+
+COPY --from=builder /graal /graal
+RUN mkdir babashka
+WORKDIR /babashka
+
+COPY src/find.clj src/find.clj
+COPY --from=builder /graal/finder bb/finder
+COPY bb.edn bb.edn
+ENV PATH="${PATH}:/graal/graalvm-ee-java19-22.3.0/bin"
+ENV JAVA_HOME="/graal/graalvm-ee-java19-22.3.0"
+
+RUN bb prepare
+WORKDIR /babashka
+ENTRYPOINT ["bb", "./src/find.clj"]

headless/bb.edn

@@ -0,0 +1,5 @@
+{
+ :paths ["src"]
+ :deps {etaoin/etaoin {:mvn/version "1.0.38"}
+        org.clojars.askonomm/ruuter {:mvn/version "1.3.2"}}
+ }

headless/main.go

@@ -0,0 +1,71 @@
+package main
+
+import "os/exec"
+import "fmt"
+import "errors"
+
+type FinderOptions struct {
+	Url string // Required
+	MatchBy string // Required, Either [ regex, css, xpath, js ]
+	Matcher string // Required, grammar/code to match against
+	Secret string // Optional, if using MatchBy regex or js, otherwise, required.
+	Timeout uint // Optional, defaults to 10 seconds.
+	Strategy string // Optional, defaults to fallback, Either [ fallback, static, webdriver ]
+}
+
+
+
+func  NewFinder(Url string, MatchBy string, Matcher string, Secret string, Timeout uint, Strategy string) (*FinderOptions, error) {
+	hop := FinderOptions{Url, MatchBy, Matcher, Secret, Timeout, Strategy}
+	if hop.Url == "" {
+		return nil, errors.New("Url must be assigned a non-empty value.")
+	}
+	if hop.MatchBy == "" || (hop.MatchBy != "regex" && hop.MatchBy != "css" && hop.MatchBy != "xpath" && hop.MatchBy != "js") {
+		return nil, errors.New(`
+           MatchBy must be assigned a valid value.
+            Valid values: [ "regex", "css", "xpath", "js" ]`)
+	}
+	if hop.Matcher == "" {
+		return nil, errors.New("Matcher required.")
+	}
+	if hop.Secret == "" && hop.MatchBy != "regex" && hop.MatchBy != "js" {
+		return nil, errors.New("Secret required.")
+	}
+	if hop.Strategy == "" || (hop.Strategy != "fallback" && hop.Strategy != "static" && hop.Strategy != "webdriver")  {
+		hop.Strategy = "fallback"
+	}
+	if hop.Timeout == 0 {
+		hop.Timeout = 5
+	}
+	if hop.MatchBy == "js" || hop.MatchBy == "xpath" {
+		hop.Strategy = "webdriver"
+	}
+	return &hop, nil
+}
+
+
+func find(opt *FinderOptions) (string) {
+	out, bberr := exec.Command("bb", "src/find.clj",
+		"--url", opt.Url,
+		"--match-by", opt.MatchBy,
+		"--matcher", opt.Matcher,
+		"--secret", opt.Secret,
+		"--timeout", string(opt.Timeout),
+		"--strategy", opt.Strategy).Output()
+	if bberr != nil {
+		fmt.Print(bberr.Error())
+	}
+	fmt.Print(string(out))
+	return string(out) // Last line contains the JSON encoded output
+
+}
+
+func main() {
+	finder_opts, arg_err := NewFinder("https://clojure.org", "css", "div.clj-header-message", "Clojure is a robust, practical, and fast programming language with a set of useful features that together form a simple, coherent, and powerful tool.", 10, "")
+	if arg_err != nil {
+		fmt.Print(arg_err.Error())
+	}
+
+	output := find(finder_opts)
+
+}

headless/src/find.clj

@@ -0,0 +1,156 @@
+(ns find (:require [babashka.pods :as pods]
+                   [babashka.cli :as cli]
+                   [etaoin.api :as e] ;; WebDriver API
+                   [ruuter.core :as ruuter] ;; HTTP router
+                   [cheshire.core :as json]
+                   [org.httpkit.server :as http]
+                   [org.httpkit.client :refer [get]]
+                   [clojure.core.match :refer [match]] ;; Pattern Matching
+                   [clojure.java.io :as io]))
+
+(pods/load-pod "./bb/finder")
+(require '[pod.jaydeesimon.jsoup :as jsoup]) ;; jsoup css selectors library
+
+(def CSS jsoup/select)
+
+(def chrome-driver-opts {:capabilities {:chromeOptions {:args ["--headless" "--no-sandbox"]}}})
+
+
+;; Finder API Strategies
+(def static-css-strategy
+  (fn [url selector]
+    (-> @(get url)
+        :body
+        (CSS selector)
+        first
+        :text)))
+
+(def static-regex-strategy
+  (fn [url regex]
+    (let [src (-> @(get url) :body)
+          ptrn (re-pattern regex)]
+          (re-find ptrn src))))
+
+(def webdriver-js-strategy
+  (fn [url js]
+    (e/with-driver :chrome chrome-driver-opts driver
+      (e/go driver url)
+      (e/js-execute driver js))))
+
+(def webdriver-xpath-strategy
+  (fn [url xpath]
+    (e/with-driver :chrome chrome-driver-opts driver
+      (e/go driver url)
+      (e/get-element-text driver {:xpath xpath}))))
+
+(def webdriver-css-strategy
+  (fn [url selector]
+    (e/with-driver :chrome chrome-driver-opts driver
+      (e/go driver url)
+      (e/get-element-text driver {:css selector}))))
+
+(def webdriver-regex-strategy
+  (fn [url regex]
+    (e/with-driver :chrome chrome-driver-opts driver
+      (e/go driver url)
+      (let [src (e/get-source driver)
+            ptrn (re-pattern regex)]
+        (re-find ptrn src)))))
+
+;; Fallback Strategies (static first, webdriver last)
+(defn fallback-regex
+  ([url matcher] (let [static-res? (boolean (static-regex-strategy url matcher))]
+                   (if static-res?
+                     true
+                     (boolean (webdriver-regex-strategy url matcher)))))
+  ([url matcher secret] (let [static-res? (= secret (static-regex-strategy url matcher))]
+                          (if static-res?
+                            true
+                            (= secret (webdriver-regex-strategy url matcher))))))
+
+(defn fallback-css [url matcher secret]
+  (let [static-res? (= secret (static-css-strategy url matcher))]
+    (if static-res?
+      true
+      (= secret (webdriver-css-strategy url matcher)))
+    ))
+
+;; Option Validation and Strategy Dispatch
+(defn dispatch [opts]
+  (when (or
+         (= "" (opts :url)) (nil? (opts :url))
+         (= "" (opts :matcher)) (nil? (opts :matcher))
+         (= "" (opts :match-by)) (nil? (opts :match-by)))
+    {:status 400 :body ((if (opts :batch?) identity json/generate-string) {:match false :message (str "[error] url, matcher, and match-by must all contain valid values.")})})
+  (let [{:keys [url matcher secret batch?]} opts
+        result (match [(merge opts (if (nil? secret) {:secret ""} {}))]
+                      [{:strategy "webdriver" :match-by "js" :secret ""}] (boolean (webdriver-js-strategy url matcher))
+                      [{:strategy "fallback" :match-by "regex" :secret ""}] (fallback-regex url matcher)
+                      [{:strategy "static" :match-by "regex" :secret ""}] (boolean (static-regex-strategy url matcher))
+                      [{:strategy "webdriver" :match-by "regex" :secret ""}] (boolean (webdriver-regex-strategy url matcher))
+                      [{:secret ""}] "Finder: Invalid configuration, secret must be non empty"
+                      [{:strategy "fallback" :match-by "css"}] (fallback-css url matcher secret)
+                      [{:strategy "fallback" :match-by "regex"}] (fallback-regex url matcher secret)
+                      [{:strategy "static" :match-by "regex"}] (= secret (static-regex-strategy url matcher))
+                      [{:strategy "static" :match-by "css"}] (= secret (static-css-strategy url matcher))
+                      [{:strategy "webdriver" :match-by "regex"}] (= secret (webdriver-regex-strategy url matcher))
+                      [{:strategy "webdriver" :match-by "css"}] (= secret (webdriver-css-strategy url matcher))
+                      [{:strategy "webdriver" :match-by "xpath"}] (= secret (webdriver-xpath-strategy url matcher))
+                      [{:strategy "webdriver" :match-by "js"}] (= secret (webdriver-js-strategy url matcher))
+                      :else "Finder: Invalid configuration, unknown." )]
+        (if (string? result)
+          {:status 400 :body ((if batch? identity json/generate-string) {:match false :message (str "[error] " result)})}
+          {:status 200 :body ((if batch? identity json/generate-string) {:match result :message (if result "match found" "no match")})})
+    ))
+
+;; Modified dispatch and find handlers for batch processing
+(defn batch-dispatch [opts]
+  (:body (dispatch (merge opts {:batch? true}))))
+
+(defn find-batch [opt-arr]
+ { :status 200 :body (json/generate-string (doall (pmap batch-dispatch opt-arr)))})
+
+;; Find api CLI & HTTP entrypoint
+(defn find [opts] (if (contains? opts :batch) (find-batch (opts :batch)) (dispatch opts)))
+
+
+;; HTTP Request middleware
+(defn json->edn [reader] (json/parse-stream reader true))
+(defn parse-json-middleware [request]
+  (-> request
+      :body
+      io/reader
+      json->edn))
+
+;; HTTP routes
+(def routes
+  [{:path "/health"
+    :method :get
+    :response {:status  200
+               :headers {"Content-Type" "text/html"}
+               :body "Ok"}}
+   {:path "/v1/find"
+    :method :get
+    :response (fn [req]
+                (-> req
+                    parse-json-middleware
+                    find))
+    }])
+
+;; CLI Server entrypoint
+(defn start-server [{:keys [port address] :or {port 8080 address "0.0.0.0"}}]
+  (http/run-server
+   #(ruuter/route routes %)
+   {:port (int port)
+    :address address})
+  (println "Server started on " address ":" port)
+  @(promise))
+
+;; Main entrypoint
+(defn -main [args] (let [cli-arg (cli/parse-opts args)]
+                       (if (contains? cli-arg :server)
+                         (start-server cli-arg)
+                         (:body (find cli-arg)))))
+(-main *command-line-args*)
+
+