Feat: publish all docs in folder

contributing.md

@@ -107,6 +107,15 @@ _(note that running mypy and black with no errors is required before code will b
 - do test coverage calculation (https://coverage.readthedocs.io/en/6.3.2)
 > bin/coverage.sh
 
+### Testing
+
+We have three levels of testing.
+1. Testing for all the little helper methods. These should be straightforward unit tests.
+
+2. "e2e" offline testing: Every command needs to have test coverage calling "MyCommand.run_command()" that will run through the basic happy path of the command. Since they are offline, they need to use the mocked server object set up in mock_data. These tests are in files named test_e2e_x_command.py, and they are run with the unit tests against every checkin.
+
+3. real, live, e2e tests that you can run against a known server when given credentials. These tests should not have any mocking code. You can launch these tests manually (see above)
+
 
 
 ### Localization

tabcmd/commands/datasources_and_workbooks/publish_command.py

@@ -1,5 +1,6 @@
 import tableauserverclient as TSC
-from tableauserverclient import ServerResponseError
+import glob
+import os
 
 from tabcmd.commands.auth.session import Session
 from tabcmd.commands.constants import Errors
@@ -24,9 +25,17 @@ def define_args(publish_parser):
         group = publish_parser.add_argument_group(title=PublishCommand.name)
         group.add_argument(
             "filename",
+            # this is a string and not actually a File type because we just pass the path to tsc
             metavar="filename.twbx|tdsx|hyper",
-            # this is not actually a File type because we just pass the path to tsc
+            help="The specified file to publish. If a folder is given, it will publish all files in this folder \
+                that have the extensions twb, twbx, tdsx or hyper. Any other options set will be applied for all files.",
         )
+        group.add_argument(
+            "--filetype",
+            metavar="twb|twbx|tdxs|hyper",
+            help="If publishing an entire folder, limit files to this filetype.",
+        )
+        group.add_argument("--recursive", help="If publishing an entire folder, look into subdirectories to find files")
         set_publish_args(group)
         set_project_r_arg(group)
         set_overwrite_option(group)
@@ -65,50 +74,76 @@ def run_command(args):
             )
         else:
             logger.debug("No db-username or oauth-username found in command")
-            connection = None
+            creds = None
+        credentials = TSC.ConnectionItem() if creds else None
+        if credentials:
+            credentials.connection_credentials = creds
+
+        files = PublishCommand.get_files_to_publish(args, logger)
+
+        logger.debug("Publishing {} files".format(len(files)))
+        for str_filename in files:
+            source = PublishCommand.get_filename_extension_if_tableau_type(logger, str_filename)
+            logger.info(_("publish.status").format(str_filename))
+            if source in ["twbx", "twb"]:
+                try:
+                    published_item = PublishCommand.publish_workbook_file(
+                        args=args,
+                        logger=logger,
+                        server=server,
+                        project_id=project_id,
+                        str_filename=str_filename,
+                        publish_mode=publish_mode,
+                        credentials=credentials,
+                    )
+                except Exception as e:
+                    Errors.exit_with_error(logger, exception=e)
+                logger.info(_("publish.success") + "\n{}".format(published_item.webpage_url))
+
+            elif source in ["tds", "tdsx", "hyper"]:
+                try:
+                    published_item = PublishCommand.publish_datasource_file(
+                        args=args,
+                        logger=logger,
+                        server=server,
+                        project_id=project_id,
+                        str_filename=str_filename,
+                        publish_mode=publish_mode,
+                        credentials=creds,
+                    )
+                except Exception as exc:
+                    Errors.exit_with_error(logger, exception=exc)
+                logger.info(_("publish.success") + "\n{}".format(published_item.webpage_url))
 
-        if connection:
-            connections = list()
-            connections.append(connection)
-        else:
-            connections = None
-
-        source = PublishCommand.get_filename_extension_if_tableau_type(logger, args.filename)
-        logger.info(_("publish.status").format(args.filename))
-        if source in ["twbx", "twb"]:
-            if args.thumbnail_username and args.thumbnail_group:
-                raise AttributeError("Cannot specify both a user and group for thumbnails.")
-
-            new_workbook = TSC.WorkbookItem(project_id, name=args.name, show_tabs=args.tabbed)
-            if args.thumbnail_username:
-                new_workbook.thumbnails_user_id = args.thumbnail_username
-            elif args.thumbnail_group:
-                new_workbook.thumbnails_group_id = args.thumbnail_group
-
-            try:
-                new_workbook = server.workbooks.publish(
-                    new_workbook,
-                    args.filename,
-                    publish_mode,
-                    connections=connections,
-                    as_job=False,
-                    skip_connection_check=args.skip_connection_check,
-                )
-            except Exception as e:
-                Errors.exit_with_error(logger, exception=e)
-
-            logger.info(_("publish.success") + "\n{}".format(new_workbook.webpage_url))
-
-        elif source in ["tds", "tdsx", "hyper"]:
-            new_datasource = TSC.DatasourceItem(project_id, name=args.name)
-            new_datasource.use_remote_query_agent = args.use_tableau_bridge
-            try:
-                new_datasource = server.datasources.publish(
-                    new_datasource, args.filename, publish_mode, connections=connections
-                )
-            except Exception as exc:
-                Errors.exit_with_error(logger, exception=exc)
-            logger.info(_("publish.success") + "\n{}".format(new_datasource.webpage_url))
+    @staticmethod
+    def get_files_to_publish(args, logger):
+        logger.debug("Checking file argument: {}".format(args.filename))
+        files = set()
+        if not os.path.exists(args.filename):
+            logger.debug("Invalid file")
+            Errors.exit_with_error(logger, message="Filename given does not exist: {}".format(args.filename))
+        elif os.path.isfile(args.filename):
+            logger.debug("Valid single file found")
+            files.add(args.filename)
+        elif os.path.isdir(args.filename):
+            logger.debug("Valid folder found")
+            if args.filetype:
+                file_patterns = [args.filetype]
+            else:
+                file_patterns = ["*.twb?", "*.tdsx", "*.hyper"]
+            logger.debug("file patterns: {}".format(file_patterns))
+            for file_pattern in file_patterns:
+                logger.debug("Looking for files {} in {}".format(file_pattern, args.filename))
+                try:
+                    in_place_files = glob.glob(
+                        file_pattern, root_dir=args.filename, recursive=args.recursive, include_hidden=False
+                    )
+                    relative_files = list(map(lambda file: os.path.join(args.filename, file), in_place_files))
+                except Exception as e:
+                    Errors.exit_with_error(logger, message=in_place_files)
+                files.update(relative_files)
+                logger.debug(len(files))
+        return sorted(files)
 
     # todo write tests for this method
     @staticmethod
@@ -133,3 +168,32 @@ def get_publish_mode(args, logger):
 
         logger.debug("Publish mode selected: " + publish_mode)
         return publish_mode
+
+    @staticmethod
+    def publish_workbook_file(args, logger, server, project_id, str_filename, publish_mode, credentials):
+        if args.thumbnail_group:
+            raise AttributeError("Generating thumbnails for a group is not yet implemented.")
+        if args.thumbnail_username and args.thumbnail_group:
+            raise AttributeError("Cannot specify both a user and group for thumbnails.")
+
+        new_workbook = TSC.WorkbookItem(project_id, name=args.name, show_tabs=args.tabbed)
+        new_workbook = server.workbooks.publish(
+            new_workbook,
+            str_filename,
+            publish_mode,
+            # args.thumbnail_username, not yet implemented in tsc
+            # args.thumbnail_group,
+            connections=credentials,
+            as_job=False,
+            skip_connection_check=args.skip_connection_check,
+        )
+        return new_workbook
+
+    @staticmethod
+    def publish_datasource_file(args, logger, server, project_id, str_filename, publish_mode, credentials):
+        new_datasource = TSC.DatasourceItem(project_id, name=args.name)
+        new_datasource.use_remote_query_agent = args.use_tableau_bridge
+        new_datasource = server.datasources.publish(
+            new_datasource, str_filename, publish_mode, connection_credentials=credentials
+        )
+        return new_datasource

tests/assets/mock_data.py

@@ -0,0 +1,115 @@
+import argparse
+import io
+
+import tableauserverclient as TSC
+from typing import NamedTuple, TextIO, Union
+from unittest.mock import *
+
+
+def create_fake_item():
+    fake_item = MagicMock()
+    fake_item.name = "fake-name"
+    fake_item.id = "fake-id"
+    fake_item.pdf = b"/pdf-representation-of-view"
+    fake_item.extract_encryption_mode = "Disabled"
+    return fake_item
+
+
+def create_fake_job():
+    fake_job = MagicMock()
+    fake_job.id = "fake-job-id"
+    return fake_job
+
+
+def set_up_mock_args():
+    mock_args = argparse.Namespace()
+    # auth/connection values
+    mock_args.timeout = None
+    mock_args.username = None
+    mock_args.server = None
+    mock_args.password_file = None
+    mock_args.token_file = None
+    mock_args.token_name = None
+    mock_args.token_value = None
+    mock_args.no_prompt = False
+    mock_args.certificate = None
+    mock_args.no_certcheck = True
+    mock_args.no_proxy = True
+    mock_args.proxy = None
+    mock_args.password = None
+    mock_args.site_name = None
+
+    # these are just really common
+    mock_args.project_name = None
+    mock_args.parent_project_path = None
+    mock_args.parent_path = None
+    mock_args.continue_if_exists = False
+    mock_args.recursive = False
+    mock_args.logging_level = "DEBUG"
+    return mock_args
+
+
+# TODO: get typings for argparse
+class NamedObject(NamedTuple):
+    name: str
+
+
+ArgparseFile = Union[TextIO, NamedObject]
+
+
+def set_up_mock_file(content=["Test", "", "Test", ""]) -> ArgparseFile:
+    # the empty string represents EOF
+    # the tests run through the file twice, first to validate then to fetch
+    mock = MagicMock(io.TextIOWrapper)
+    mock.readline.side_effect = content
+    mock.name = "file-mock"
+    return mock
+
+
+def set_up_mock_path(mock_path):
+    mock_path.exists = lambda x: True
+    mock_path.isfile = lambda x: True
+    mock_path.isdir = lambda x: True
+    mock_path.splitext = lambda x: ["file", "twbx"]
+    mock_path.join = lambda x, y: x + "/" + y
+    mock_path.basename = lambda x: str(x)
+    return mock_path
+
+
+def set_up_mock_server(mock_session):
+
+    mock_session.return_value = mock_session
+    mock_server = MagicMock(TSC.Server, autospec=True)
+    getter = MagicMock()
+    # basically we want to mock out everything in TSC
+    # Return a pagination-like object with required attributes
+    fake_pagination = MagicMock()
+    fake_pagination.total_available = 1
+    fake_pagination.page_number = 1
+    fake_pagination.page_size = 100
+    getter.get = MagicMock("get anything", return_value=([create_fake_item()], fake_pagination))
+    getter.publish = MagicMock("publish", return_value=create_fake_item())
+
+    mock_server.any_item_type = getter
+    mock_server.flows = getter
+    mock_server.groups = getter
+    mock_server.projects = getter
+    mock_server.sites = getter
+    mock_server.users = getter
+    mock_server.views = getter
+    mock_server.workbooks = getter
+
+    fake_job = create_fake_job()
+    # ideally I would only set these on the specific objects that have each action, but this is a start
+    getter.create_extract = MagicMock("create_extract", return_value=fake_job)
+    getter.decrypt_extract = MagicMock("decrypt_extract", return_value=fake_job)
+    getter.delete_extract = MagicMock("delete_extract", return_value=fake_job)
+    getter.encrypt_extracts = MagicMock("encrypt_extracts", return_value=fake_job)
+    getter.reencrypt_extract = MagicMock("reencrypt_extract", return_value=fake_job)
+    getter.refresh = MagicMock("refresh", return_value=fake_job)
+
+    # for test access
+    mock_session.internal_server = mock_server
+    mock_session.create_session.return_value = mock_server
+
+    return mock_session