diff --git a/.gitignore b/.gitignore index b5450c6c5..eea46d9e9 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,6 @@ /.idea/ .venv/ -build +/build/ cmake/sodium_version cmake/curl_version cmake/zlib_version diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d3c5504aa..7e68ba465 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -26,6 +26,7 @@ include: - local: .gitlab/stage_test.yml - local: .gitlab/stage_unit.yml - local: .gitlab/end_to_end.yml + - local: .gitlab/stage_deploy_pypi.yml stages: - pipeline-serialize diff --git a/.gitlab/build/build_python_client_image.yml b/.gitlab/build/build_python_client_image.yml new file mode 100644 index 000000000..821fdad3f --- /dev/null +++ b/.gitlab/build/build_python_client_image.yml @@ -0,0 +1,48 @@ +--- +stages: + - build + +include: + - local: .gitlab/common.yml + +build-python-client: + extends: .docker_build_script + stage: build + variables: + PROJECT: "datafed" + COMPONENT: "python-client" + GIT_STRATEGY: clone + DOCKER_FILE_PATH: "python/datafed_pkg/docker/Dockerfile" + DATAFED_HARBOR_REGISTRY: "$REGISTRY" # needed by c_harbor_artifact_count + BUILD_INTERMEDIATE: "FALSE" + tags: + - docker + rules: + - changes: + - python/**/* + - common/**/* + - CMakeLists.txt + - cmake/**/* + - .gitlab-ci.yml + when: on_success + +retag-image: + extends: .docker_retag_image + stage: build + variables: + PROJECT: "datafed" + COMPONENT: "python-client" + GIT_STRATEGY: clone + DATAFED_HARBOR_REGISTRY: "$REGISTRY" # needed by c_harbor_artifact_count + BUILD_INTERMEDIATE: "FALSE" + tags: + - docker + rules: + - changes: + - python/**/* + - common/**/* + - CMakeLists.txt + - cmake/**/* + - .gitlab-ci.yml + when: never + - when: on_success diff --git a/.gitlab/build/force_build_python_client_image.yml b/.gitlab/build/force_build_python_client_image.yml new file mode 100644 index 000000000..75c9cfdd3 --- /dev/null +++ b/.gitlab/build/force_build_python_client_image.yml @@ -0,0 +1,19 @@ +--- +stages: + - build + +include: + - local: .gitlab/common.yml + +build-python-client: + extends: .docker_build_script + stage: build + variables: + PROJECT: "datafed" + COMPONENT: "python-client" + GIT_STRATEGY: clone + DOCKER_FILE_PATH: "python/datafed_pkg/docker/Dockerfile" + DATAFED_HARBOR_REGISTRY: "$REGISTRY" # needed by c_harbor_artifact_count + BUILD_INTERMEDIATE: "FALSE" + tags: + - docker diff --git a/.gitlab/stage_build.yml b/.gitlab/stage_build.yml index 52a4c1c12..dbc2ab2e7 100644 --- a/.gitlab/stage_build.yml +++ b/.gitlab/stage_build.yml @@ -102,3 +102,19 @@ run-foxx-build-job: REGISTRY: "${REGISTRY}" HARBOR_USER: "${HARBOR_USER}" HARBOR_DATAFED_GITLAB_CI_REGISTRY_TOKEN: "${HARBOR_DATAFED_GITLAB_CI_REGISTRY_TOKEN}" + +run-python-client-build-job: + needs: + - job: run-build-dependencies + - job: check-python-client-image + artifacts: true + stage: build + trigger: + include: + - artifact: python_client_image.yml + job: check-python-client-image + strategy: depend + variables: + REGISTRY: "${REGISTRY}" + HARBOR_USER: "${HARBOR_USER}" + HARBOR_DATAFED_GITLAB_CI_REGISTRY_TOKEN: "${HARBOR_DATAFED_GITLAB_CI_REGISTRY_TOKEN}" diff --git a/.gitlab/stage_deploy_pypi.yml b/.gitlab/stage_deploy_pypi.yml new file mode 100644 index 000000000..005d77bb5 --- /dev/null +++ b/.gitlab/stage_deploy_pypi.yml @@ -0,0 +1,92 @@ +# GitLab CI configuration for deploying DataFed Python package to PyPI +# +# This job builds the Python package fresh and deploys it to PyPI using twine. +# The version numbers are automatically determined from cmake/Version.cmake. +# +# Required GitLab CI/CD Variables: +# - PYPI_PASSWORD: PyPI password or API token +# +# Optional Variables (with defaults): +# - PYPI_USERNAME: PyPI username (default: "__token__" for API token auth) +# - PYPI_PRERELEASE: If "true", append pre-release identifier (default: "true") +# - PYPI_REPOSITORY_URL: PyPI repository URL (default: "https://upload.pypi.org/legacy/" for production PyPI) +# Use "https://test.pypi.org/legacy/" for TestPyPI +# - DATAFED_PYPI_REPO: PyPI package name (default: "datafed") +# Use a different name like "datafed-dev" to upload to a different package on PyPI + +.deploy_pypi_base: + stage: deploy-pypi-package + variables: + PYPI_USERNAME: "__token__" + PYPI_PRERELEASE: "true" + PYPI_REPOSITORY_URL: "https://upload.pypi.org/legacy/" + DATAFED_PYPI_REPO: "datafed" + before_script: + - docker login "${REGISTRY}" -u "${HARBOR_USER}" -p "${HARBOR_DATAFED_GITLAB_CI_REGISTRY_TOKEN}" + script: + - DOWNSTREAM_SHA=$( git submodule status ./external/DataFedDependencies/ | awk '{print $1}' ) + - DOWNSTREAM_SHA=${DOWNSTREAM_SHA#-} + - mkdir -p dist + + # Modify Version.cmake to add pre-release identifier if needed + - | + if [ "${PYPI_PRERELEASE}" = "true" ]; then + echo "Building package with pre-release identifier (rc${CI_PIPELINE_IID})..." + sed -i "s/set(DATAFED_PYTHON_CLIENT_RELEASE_TYPE \"\")/set(DATAFED_PYTHON_CLIENT_RELEASE_TYPE \"rc\")/" cmake/Version.cmake + sed -i "s/set(DATAFED_PYTHON_CLIENT_PRE_RELEASE_IDENTIFER \"\")/set(DATAFED_PYTHON_CLIENT_PRE_RELEASE_IDENTIFER \"${CI_PIPELINE_IID}\")/" cmake/Version.cmake + else + echo "Building package as full release..." + fi + + # Build the Python package using existing Dockerfile + - | + docker build \ + --build-arg DEPENDENCIES="${REGISTRY}/datafed/dependencies:$DOWNSTREAM_SHA" \ + --build-arg DATAFED_PYPI_REPO="${DATAFED_PYPI_REPO}" \ + -f python/datafed_pkg/docker/Dockerfile \ + -t datafed-python-client-deploy:latest \ + . + + # Extract packages + - docker run --rm -v "$(pwd)/dist:/output" datafed-python-client-deploy:latest sh -c "cp /dist/* /output/" + + # Restore Version.cmake to original state if modified + - | + if [ "${PYPI_PRERELEASE}" = "true" ]; then + git checkout cmake/Version.cmake + fi + + # Verify packages exist + - ls -lh dist/ + + # Upload to PyPI using twine + - | + docker run --rm \ + -v "$(pwd)/dist:/dist" \ + -e PYPI_USERNAME="${PYPI_USERNAME}" \ + -e PYPI_PASSWORD="${PYPI_PASSWORD}" \ + -e PYPI_REPOSITORY_URL="${PYPI_REPOSITORY_URL}" \ + datafed-python-client-deploy:latest \ + sh -c "twine upload --non-interactive --repository-url \$PYPI_REPOSITORY_URL --username \$PYPI_USERNAME --password \$PYPI_PASSWORD /dist/*" + artifacts: + paths: + - dist/ + expire_in: 1 week + tags: + - docker + +# Manual deployment job - can be triggered via button in GitLab UI +deploy:pypi:manual: + extends: .deploy_pypi_base + when: manual + rules: + - when: manual + +# Automatic deployment on tags (optional - commented out by default) +# Uncomment to enable automatic deployment when tags are pushed +# deploy:pypi:tag: +# extends: .deploy_pypi_base +# needs: +# - job: run-python-client-build-job +# only: +# - tags diff --git a/.gitlab/stage_image_check.yml b/.gitlab/stage_image_check.yml index 10028b407..485b69b66 100644 --- a/.gitlab/stage_image_check.yml +++ b/.gitlab/stage_image_check.yml @@ -50,3 +50,11 @@ check-foxx-image: PROJECT: "datafed" COMPONENT: "foxx" BUILD_INTERMEDIATE: "FALSE" + +check-python-client-image: + extends: .image_check + stage: image-check + variables: + PROJECT: "datafed" + COMPONENT: "python_client" + BUILD_INTERMEDIATE: "FALSE" diff --git a/CMakeLists.txt b/CMakeLists.txt index ac472ce8e..e04ea9cb6 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -200,7 +200,7 @@ endif() if( BUILD_PYTHON_CLIENT ) # make target = pydatafed - file(COPY ${PROJECT_SOURCE_DIR}/external/DataFedDependencies/python/datafed_pkg/requirements.txt DESTINATION ${PROJECT_SOURCE_DIR}/python/datafed_pkg/requirements.txt) + file(COPY ${PROJECT_SOURCE_DIR}/external/DataFedDependencies/python/datafed_pkg/requirements.txt DESTINATION ${PROJECT_SOURCE_DIR}/python/datafed_pkg/) add_subdirectory( python EXCLUDE_FROM_ALL ) endif() diff --git a/docs/_static/js/html5shiv-printshiv.min.js b/docs/_static/js/html5shiv-printshiv.min.js index 72406b792..ae10bdff2 100644 --- a/docs/_static/js/html5shiv-printshiv.min.js +++ b/docs/_static/js/html5shiv-printshiv.min.js @@ -88,7 +88,6 @@ f = RegExp("^(?:" + d().join("|") + ")$", "i"), g = []; e--; - ) ((b = c[e]), f.test(b.nodeName) && g.push(b.applyElement(l(b)))); return g; @@ -100,7 +99,6 @@ d = c.length, e = a.ownerDocument.createElement(A + ":" + a.nodeName); d--; - ) ((b = c[d]), b.specified && e.setAttribute(b.nodeName, b.nodeValue)); return ((e.style.cssText = a.style.cssText), e); @@ -113,7 +111,6 @@ f = RegExp("(^|[\\s,>+~])(" + d().join("|") + ")(?=[[\\s,>+~#.:]|$)", "gi"), g = "$1" + A + "\\:$2"; e--; - ) ((b = c[e] = c[e].split("}")), (b[b.length - 1] = b[b.length - 1].replace(f, g)), diff --git a/python/datafed_pkg/docker/Dockerfile b/python/datafed_pkg/docker/Dockerfile new file mode 100644 index 000000000..1bc3c7968 --- /dev/null +++ b/python/datafed_pkg/docker/Dockerfile @@ -0,0 +1,93 @@ +# NOTE this image must be built with respect to the base of the project i.e. +# cd ${PROJECT_ROOT} or cd DataFed +# docker build -f python/datafed_pkg/docker/Dockerfile . + +ARG BUILD_BASE="debian:bookworm-slim" +ARG DEPENDENCIES="dependencies" +ARG DATAFED_DIR="/datafed" +ARG DATAFED_INSTALL_PATH="/opt/datafed" +ARG DATAFED_DEPENDENCIES_INSTALL_PATH="/opt/datafed/dependencies" +ARG BUILD_DIR="$DATAFED_DIR/source" +ARG DATAFED_DEPENDENCIES_ROOT="$BUILD_DIR/external/DataFedDependencies" +ARG DATAFED_PYPI_REPO="datafed" + +FROM ${DEPENDENCIES} AS python-client-build + +SHELL ["/bin/bash", "-c"] + +ARG DATAFED_DIR +ARG BUILD_DIR +ARG DATAFED_INSTALL_PATH +ARG DATAFED_DEPENDENCIES_INSTALL_PATH +ARG DATAFED_DEPENDENCIES_ROOT +ARG DATAFED_PYPI_REPO + +ENV DATAFED_INSTALL_PATH="${DATAFED_INSTALL_PATH}" +ENV DATAFED_DEPENDENCIES_INSTALL_PATH="${DATAFED_DEPENDENCIES_INSTALL_PATH}" +ENV DATAFED_PYPI_REPO="${DATAFED_PYPI_REPO}" + +RUN mkdir -p ${DATAFED_DEPENDENCIES_ROOT}/scripts/ && \ + mv ./scripts/dependency_versions.sh ${DATAFED_DEPENDENCIES_ROOT}/scripts/ && \ + mv ./scripts/generate_dependencies_config.sh ${DATAFED_DEPENDENCIES_ROOT}/scripts/ + +COPY ./common ${BUILD_DIR}/common +COPY ./python ${BUILD_DIR}/python +COPY ./CMakeLists.txt ${BUILD_DIR} +COPY ./scripts/generate_datafed.sh ${BUILD_DIR}/scripts/ +COPY ./cmake ${BUILD_DIR}/cmake +COPY ./external/DataFedDependencies ${BUILD_DIR}/external/DataFedDependencies + +# Configure and build the Python package (generates protobuf files and VERSION.py) +RUN ${DATAFED_DEPENDENCIES_ROOT}/scripts/generate_dependencies_config.sh && \ + ${BUILD_DIR}/scripts/generate_datafed.sh && \ + ${DATAFED_DEPENDENCIES_INSTALL_PATH}/bin/cmake -S. -B build \ + -DBUILD_REPO_SERVER=False \ + -DBUILD_AUTHZ=False \ + -DBUILD_CORE_SERVER=False \ + -DBUILD_WEB_SERVER=False \ + -DBUILD_DOCS=False \ + -DBUILD_PYTHON_CLIENT=True \ + -DBUILD_FOXX=False \ + -DENABLE_INTEGRATION_TESTS=False + +RUN ${DATAFED_DEPENDENCIES_INSTALL_PATH}/bin/cmake --build build --target pydatafed + +# Install Python build tools and package dependencies +RUN apt-get update && apt-get install -y python3-pip && rm -rf /var/lib/apt/lists/* +RUN pip3 install --no-cache-dir --upgrade pip setuptools wheel twine --break-system-packages + +# Install package dependencies from requirements.txt +WORKDIR ${BUILD_DIR}/python/datafed_pkg +RUN pip3 install --no-cache-dir -r requirements.txt --break-system-packages + +# Navigate to the package build directory and copy requirements.txt +WORKDIR ${BUILD_DIR}/build/python/datafed_pkg +RUN cp ${BUILD_DIR}/python/datafed_pkg/requirements.txt . + +# Clean any previous builds and build distributions +RUN rm -rf dist/ build/ *.egg-info/ && \ + python3 setup.py sdist bdist_wheel + +# Create output directory for artifacts +RUN mkdir -p /output && cp -r dist/* /output/ + +# Final stage for deployment - minimal image with just the built packages +FROM ${BUILD_BASE} AS python-client-deploy + +ARG DATAFED_DIR +ARG BUILD_DIR + +# Install Python and twine for PyPI upload +RUN apt-get update && \ + apt-get install -y python3-pip && \ + rm -rf /var/lib/apt/lists/* + +RUN pip3 install --no-cache-dir --upgrade pip twine --break-system-packages + +# Copy built distributions from build stage +COPY --from=python-client-build /output /dist + +WORKDIR /dist + +# The deploy command will be run via CI +CMD ["/bin/bash"] diff --git a/scripts/build_python_package.sh b/scripts/build_python_package.sh new file mode 100644 index 000000000..a40972126 --- /dev/null +++ b/scripts/build_python_package.sh @@ -0,0 +1,97 @@ +#!/bin/bash +set -e + +# Build script for DataFed Python package using Docker +# This script builds the Python package inside a Docker container using DataFed dependencies +# It reads version numbers from cmake/Version.cmake +# +# Environment variables: +# - REGISTRY: Docker registry URL (e.g., camden.ornl.gov). If set, pulls dependencies from registry. +# - HARBOR_USER: Username for registry login (required if REGISTRY is set) +# - HARBOR_DATAFED_GITLAB_CI_REGISTRY_TOKEN: Token for registry login (required if REGISTRY is set) + +echo "========================================" +echo "Building DataFed Python Package (Docker)" +echo "========================================" + +# Get the script directory +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)" + +echo "Project root: ${PROJECT_ROOT}" +cd "${PROJECT_ROOT}" + +# Get the dependencies submodule SHA for the DEPENDENCIES build arg +DEPENDENCIES_SHA=$(git submodule status ./external/DataFedDependencies/ | awk '{print $1}' | sed 's/^-//') +echo "Using dependencies SHA: ${DEPENDENCIES_SHA}" + +# Determine dependencies image source +DEPENDENCIES_IMAGE="" +if [ -n "${REGISTRY:-}" ]; then + # CI environment - pull from registry + echo "" + echo "Registry detected: ${REGISTRY}" + DEPENDENCIES_IMAGE="${REGISTRY}/datafed/dependencies:${DEPENDENCIES_SHA}" + + if [ -n "${HARBOR_USER:-}" ] && [ -n "${HARBOR_DATAFED_GITLAB_CI_REGISTRY_TOKEN:-}" ]; then + echo "Logging in to registry..." + docker login "${REGISTRY}" -u "${HARBOR_USER}" -p "${HARBOR_DATAFED_GITLAB_CI_REGISTRY_TOKEN}" + fi + + echo "Pulling dependencies image from registry..." + docker pull "${DEPENDENCIES_IMAGE}" +else + # Local development - build or use local image + echo "" + echo "No registry specified - using local development mode" + DEPENDENCIES_IMAGE="datafed-dependencies:${DEPENDENCIES_SHA}" + + echo "Checking for dependencies image..." + if ! docker image inspect datafed-dependencies:latest >/dev/null 2>&1; then + echo "Dependencies image not found locally. Building dependencies image..." + "${PROJECT_ROOT}/external/DataFedDependencies/scripts/build_image.sh" + # Tag with SHA for consistency + docker tag datafed-dependencies:latest "${DEPENDENCIES_IMAGE}" + else + echo "Dependencies image found: datafed-dependencies:latest" + # Ensure SHA-tagged version exists + if ! docker image inspect "${DEPENDENCIES_IMAGE}" >/dev/null 2>&1; then + docker tag datafed-dependencies:latest "${DEPENDENCIES_IMAGE}" + fi + fi +fi + +# Build the Python client Docker image +echo "" +echo "Building Python client Docker image..." +echo "Using dependencies: ${DEPENDENCIES_IMAGE}" +docker build \ + --build-arg DEPENDENCIES="${DEPENDENCIES_IMAGE}" \ + -f python/datafed_pkg/docker/Dockerfile \ + -t datafed-python-client:latest \ + . + +# Extract the built packages from the Docker image +echo "" +echo "Extracting built packages from Docker image..." +OUTPUT_DIR="${PROJECT_ROOT}/python/datafed_pkg/dist" +mkdir -p "${OUTPUT_DIR}" + +# Create a temporary container and copy the dist files +CONTAINER_ID=$(docker create datafed-python-client:latest) +docker cp "${CONTAINER_ID}:/dist/." "${OUTPUT_DIR}/" +docker rm "${CONTAINER_ID}" + +echo "" +echo "========================================" +echo "Build completed successfully!" +echo "========================================" +echo "Distributions created in: ${OUTPUT_DIR}/" +ls -lh "${OUTPUT_DIR}/" + +echo "" +echo "To deploy to PyPI, run:" +echo " pip install twine" +echo " twine upload ${OUTPUT_DIR}/*" + +exit 0 diff --git a/web/static/ace/worker-coffee.js b/web/static/ace/worker-coffee.js index 2692b5bca..7400d29a4 100644 --- a/web/static/ace/worker-coffee.js +++ b/web/static/ace/worker-coffee.js @@ -1646,7 +1646,6 @@ for ( i = this.tokens, n = 0; (r = i[n]); - ) n += t.call(this, r, n, i); return !0; @@ -1668,7 +1667,6 @@ for ( h = this.tokens, o = 0; (c = h[n]); - ) { if ( 0 === o && @@ -1914,7 +1912,6 @@ .generated) && ((c = this.tag(n)), 0 > t.call(v, c)))); - ) (((s = this.tag(n)), 0 <= t.call(a, s)) && @@ -2287,7 +2284,6 @@ (C() && ":" !== H); - ) T() ? g() @@ -2746,7 +2742,6 @@ ? 1 : 0; C(); - ) y(o + P); return w(1); @@ -2813,7 +2808,6 @@ u !== i.length && ((l = i[u][0]), 0 <= t.call(o, l)); - ) u++; if ( @@ -2852,7 +2846,6 @@ -1 !== s && ((u = i[s][0]), 0 <= t.call(o, u)); - ) s--; return -1 === s || @@ -2892,7 +2885,6 @@ .length - 1; -1 !== a; - ) (!1 === e @@ -2946,7 +2938,6 @@ .length - 1; -1 !== a; - ) (!e .comments[ @@ -3986,7 +3977,6 @@ t = this.clean(t), s = 0; (this.chunk = t.slice(s)); - ) { r = this.identifierToken() || @@ -4536,7 +4526,6 @@ return n; })().join("#{}"); (p = E.exec(u)); - ) ((s = p[1]), (null === c || @@ -5377,7 +5366,6 @@ (null == (i = this.ends[n]) ? void 0 : i.tag) || 0 < t--; - ) n--; return ( @@ -5558,7 +5546,6 @@ n = s[--t], n[0] = "PARAM_END"; (i = s[--t]); - ) switch (i[0]) { case ")": @@ -25711,7 +25698,6 @@ o, ) ); - ) i++; return ( @@ -26488,7 +26474,6 @@ for ( t = this; t !== (t = t.unwrap()); - ) continue; return t; @@ -26767,7 +26752,6 @@ for ( r = this.expressions.length; r--; - ) { ((n = this.expressions[r]), (this.expressions[r] = @@ -26926,8 +26910,8 @@ this, null, null == - (o = - t.referencedVars) + (o = + t.referencedVars) ? [] : o, ), @@ -30954,7 +30938,6 @@ ); }; (t = y[r]); - ) ((c = this.addInitializerExpression( @@ -31327,7 +31310,6 @@ r instanceof Pt && r.isString() ); - ) if (r.hoisted) i++; else { @@ -32965,7 +32947,8 @@ .properties[0] .name : new ct( - s.unwrap().value, + s.unwrap() + .value, )), (f = c.unwrap() instanceof @@ -34754,7 +34737,8 @@ ? r.expressions.unshift( new D( new ft( - this.guard, + this + .guard, ).invert(), new wt( "continue", @@ -34764,7 +34748,8 @@ : this.guard && (r = f.wrap([ new D( - this.guard, + this + .guard, r, ), ]))), @@ -34946,7 +34931,6 @@ for ( t = !0, n = this; n && n.operator; - ) (t && (t = @@ -34960,7 +34944,6 @@ for ( n = this; n && n.operator; - ) ((n.invert = !n.invert), (n.operator = @@ -35671,7 +35654,8 @@ : void 0, this.recovery.unshift( new o( - this.errorVariable, + this + .errorVariable, u, ), )) @@ -36629,7 +36613,8 @@ ? s.expressions.unshift( new D( new ft( - this.guard, + this + .guard, ).invert(), new wt( "continue", @@ -37370,7 +37355,6 @@ for ( var n; !((n = this.columns[t]) || 0 >= t); - ) t--; return n && [n.sourceLine, n.sourceColumn]; @@ -37422,7 +37406,6 @@ (s = this.lines[r]) || 0 >= r ); - ) r--; return s && s.sourceLocation(i); @@ -37544,7 +37527,6 @@ a = 0 > t ? 1 : 0, f = (_Mathabs(t) << 1) + a; f || !n; - ) ((u = f & s), (f >>= i),