fix: rename TestColumn classes to MockColumn to fix pytest collection warnings

.github/workflows/publish-docs.yml

@@ -0,0 +1,47 @@
+# This workflow builds and publishes the latest docs to
+# the `gh-pages` branch.
+# For more details: https://github.com/marketplace/actions/deploy-to-github-pages
+name: Publish docs
+
+on:
+  release:
+    types: [created]
+  workflow_dispatch:
+
+jobs:
+  build-and-deploy:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+      pages: write
+    defaults:
+      run:
+        shell: bash -l {0}
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        # fetch all tags so `versioneer` can properly determine current version
+        fetch-depth: 0
+    - name: Set up Python
+      uses: actions/setup-python@v2
+      with:
+        python-version: '3.11'
+    - name: Install pandoc
+      uses: pandoc/actions/setup@v1
+    - name: Install dependencies
+      run: |
+        pip install -r requirements.txt
+        pip install -r requirements-ml.txt
+        pip install -r requirements-reports.txt
+        pip install -r requirements-docs.txt
+        pip install -e .
+
+    - name: Build
+      run: |
+        cd _docs/docs
+        python update_documentation.py
+    - name: Publish
+      uses: JamesIves/github-pages-deploy-action@v4
+      with:
+        branch: gh-pages
+        folder: _docs/docs/LATEST/html

.github/workflows/publish-package.yml

@@ -0,0 +1,52 @@
+# This workflow publishes the package to pypi.
+# For more details:
+# https://docs.github.com/en/actions/guides/building-and-testing-python#publishing-to-package-registries
+name: Publish to PyPi
+
+on:
+  release:
+    types: [created]
+  workflow_dispatch:
+
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+      # fetch all tags so `versioneer` can properly determine current version
+      with:
+        fetch-depth: 0
+    - name: Check if current commit is tagged
+      # fails and cancels release if the current commit is not tagged
+      run: |
+        git describe --exact-match --tags
+    - name: Set up Python
+      uses: actions/setup-python@v2
+      with:
+        python-version: '3.11'
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+        if [ -f requirements-ml.txt ]; then pip install -r requirements-ml.txt; fi
+        if [ -f requirements-reports.txt ]; then pip install -r requirements-reports.txt; fi
+        pip install setuptools wheel twine
+    - name: Build
+      env:
+        TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }}
+        TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
+        TWINE_REPOSITORY: pypi
+      run: |
+        python setup.py sdist bdist_wheel
+    - name: Test build
+      # fails and cancels release if the built package fails to import
+      run: |
+        pip install dist/*.whl
+        python -c 'import dataprofiler; print(dataprofiler.__version__)'
+    - name: Publish
+      env:
+        TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }}
+        TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
+        TWINE_REPOSITORY: pypi
+      run: |
+        twine upload dist/*

.github/workflows/test-python-package.yml → .github/workflows/test-package.yml

@@ -7,16 +7,14 @@ on:
   pull_request:
     branches:
       - 'main'
-      - 'feature/**'
-      - 'dev'
 
 jobs:
   build:
 
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [3.9, "3.10", "3.11"]
+        python-version: ["3.10", "3.11"]
 
     steps:
     - uses: actions/checkout@v4
@@ -38,4 +36,4 @@ jobs:
         pre-commit run --all-files
     - name: Test with pytest
       run: |
-        DATAPROFILER_SEED=0 pytest --forked --cov=dataprofiler --cov-fail-under=80
+        DATAPROFILER_SEED=0 pytest --cov=dataprofiler --cov-fail-under=80

.pre-commit-config.yaml

@@ -5,6 +5,7 @@ repos:
     rev: 24.10.0
     hooks:
       - id: black
+        exclude: (versioneer.py|dataprofiler/_version.py|_docs/)
         types: [file, python]
         language_version: python3
   # Isort: sort import statements
@@ -15,6 +16,7 @@ repos:
     rev: 5.12.0
     hooks:
       - id: isort
+        exclude: _docs/
         language_version: python3
   # Flake8: complexity and style checking
   # https://flake8.pycqa.org/en/latest/user/using-hooks.html
@@ -23,25 +25,25 @@ repos:
     hooks:
       - id: flake8
         additional_dependencies: [flake8-docstrings]
-        exclude: (^docs/|^dataprofiler/tests/|^.*/__init__.py)
+        exclude: (^docs/|^dataprofiler/tests/|^.*/__init__.py|_docs/)
         language_version: python3
   # General fixers: format files for white spaces and trailing new lines, warn on debug statements
   # https://github.com/pre-commit/pre-commit-hooks#hooks-available
   - repo: https://github.com/pre-commit/pre-commit-hooks
     rev: v4.0.1
     hooks:
       - id: trailing-whitespace
-        exclude: (^dataprofiler/tests/data/|^dataprofiler/tests/speed_tests/data/)
+        exclude: (^dataprofiler/tests/data/|^dataprofiler/tests/speed_tests/data/|_docs/)
       - id: debug-statements
       - id: end-of-file-fixer
-        exclude: (^dataprofiler/tests/data/)
+        exclude: (^dataprofiler/tests/data/|_docs/)
   # Mypy: Optional static type checking
   # https://github.com/pre-commit/mirrors-mypy
   - repo: https://github.com/pre-commit/mirrors-mypy
     rev: v1.11.2
     hooks:
       - id: mypy
-        exclude: (^dataprofiler/tests/|^resources/|^examples|venv*/)
+        exclude: (^dataprofiler/tests/|^resources/|^examples|venv*/|versioneer.py|dataprofiler/_version.py|_docs/)
         language_version: python3
         additional_dependencies: # Keep up-to-date with the respective requirement files
           [
@@ -93,7 +95,6 @@ repos:
 
             # requirements-test.txt
             coverage>=5.0.1,
-            dask>=2.29.0,
             fsspec>=0.3.3,
             pytest>=6.0.1,
             pytest-cov>=2.8.1,
@@ -108,24 +109,23 @@ repos:
     rev: "0.48"
     hooks:
       - id: check-manifest
-        additional_dependencies:
-          [
-            'matplotlib', 'h5py', 'wheel', 'future', 'numpy<2.0.0', 'pandas',
-            'python-dateutil', 'pytz', 'pyarrow', 'chardet', 'fastavro',
-            'python-snappy', 'charset-normalizer', 'psutil', 'scipy', 'requests',
-            'networkx','typing-extensions', 'HLL', 'datasketches', 'boto3',
-          ]
+        additional_dependencies: ['h5py', 'wheel', 'future', 'numpy<2.0.0', 'pandas',
+        'python-dateutil', 'pytz', 'pyarrow', 'chardet', 'fastavro',
+        'python-snappy', 'charset-normalizer', 'psutil', 'scipy', 'requests',
+        'networkx','typing-extensions', 'HLL', 'datasketches', 'boto3']
   # Pyupgrade - standardize and modernize Python syntax for newer versions of the language
   - repo: https://github.com/asottile/pyupgrade
     rev: v3.3.0
     hooks:
       - id: pyupgrade
-        args: ["--py39-plus"]
+        args: ["--py310-plus"]
+        exclude: (versioneer.py|dataprofiler/_version.py| _docs/)
   # Autoflake - cleanup unused variables and imports
   - repo: https://github.com/PyCQA/autoflake
     rev: v2.0.0
     hooks:
       - id: autoflake
+        exclude: _docs/
         args:
           - "--in-place"
           - "--ignore-pass-statements"

.whitesource

@@ -0,0 +1,3 @@
+{
+    "settingsInheritedFrom": "capitalone/whitesource-config"
+}

MANIFEST.in

@@ -17,4 +17,17 @@ recursive-include resources *.json
 recursive-include resources *.pb
 recursive-include resources *.py
 
-recursive-include dataprofiler/labelers/embeddings/*.txt
+recursive-include dataprofiler/labelers/embeddings *.txt
+include versioneer.py
+include dataprofiler/_version.py
+include .whitesource
+
+recursive-exclude _docs *.html
+recursive-exclude _docs *.cfg
+exclude _docs/LICENSE
+recursive-exclude _docs *.md
+recursive-exclude _docs *.nojekyll
+recursive-exclude _docs *.png
+recursive-exclude _docs *.py
+recursive-exclude _docs *.rst
+recursive-exclude _docs Makefile

_docs/README.md

@@ -0,0 +1,59 @@
+Visit our [documentation page.](https://capitalone.github.io/DataProfiler)
+
+### How to properly write documentation:
+
+#### Packages
+In any package directory, overall package comments can be made in the
+\_\_init\_\_.py of the directory. At the top of the \_\_init\_\_.py,
+include your comments in between triple quotations.
+
+#### Classes
+In any class file, include overall class comments at the top of the file
+in between triple quotes and/or in the init function.
+
+#### Functions
+reStructuredText Docstring Format is the standard. Here is an example:
+
+    def format_data(self, predictions, verbose=False):
+        """
+        Formats word level labeling of the Unstructured Data Labeler as you want
+
+        :param predictions: A 2D list of word level predictions/labeling
+        :type predictions: Dict
+        :param verbose: A flag to determine verbosity
+        :type verbose: Bool
+        :return: JSON structure containing specified formatted output
+        :rtype: JSON
+
+        :Example:
+            Look at this test. Don't forget the double colons to make a code block::
+                This is a codeblock
+                Type example code here
+        """
+
+### How to update the documentation:
+
+
+1. Set up your local environment
+```bash
+# install sphinx requirements
+# install the requirements from the feature branch
+pip install pandoc &&
+pip install -r requirements.txt &&
+pip install -r requirements-ml.txt && 
+pip install -r requirements-reports.txt && 
+pip install -r requirements-docs.txt  &&
+pip install -e . 
+
+```
+2. And finally, from the root of `DataProfiler`, run the following commands to generate the sphinx documentation:
+```bash
+cd _docs/docs
+python update_documentation.py
+
+```
+
+3. View new docs
+```bash
+open index.html
+```

_docs/docs/Makefile

@@ -0,0 +1,20 @@
+# Minimal makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line, and also
+# from the environment for the first two.
+SPHINXOPTS    ?=
+SPHINXBUILD   ?= sphinx-build
+SOURCEDIR     = source
+BUILDDIR      = buildcode
+
+# Put it first so that "make" without argument is like "make help".
+help:
+	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+.PHONY: help Makefile
+
+# Catch-all target: route all unknown targets to Sphinx using the new
+# "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
+%: Makefile
+	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

_docs/docs/make.bat

@@ -0,0 +1,35 @@
+@ECHO OFF
+
+pushd %~dp0
+
+REM Command file for Sphinx documentation
+
+if "%SPHINXBUILD%" == "" (
+	set SPHINXBUILD=sphinx-build
+)
+set SOURCEDIR=source
+set BUILDDIR=build
+
+if "%1" == "" goto help
+
+%SPHINXBUILD% >NUL 2>NUL
+if errorlevel 9009 (
+	echo.
+	echo.The 'sphinx-build' command was not found. Make sure you have Sphinx
+	echo.installed, then set the SPHINXBUILD environment variable to point
+	echo.to the full path of the 'sphinx-build' executable. Alternatively you
+	echo.may add the Sphinx directory to PATH.
+	echo.
+	echo.If you don't have Sphinx installed, grab it from
+	echo.http://sphinx-doc.org/
+	exit /b 1
+)
+
+%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
+goto end
+
+:help
+%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
+
+:end
+popd