-
Notifications
You must be signed in to change notification settings - Fork 772
[#9882] Add implementation for TagOperations in python client(part2) #10554
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: main
Are you sure you want to change the base?
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -15,6 +15,10 @@ | |
| # specific language governing permissions and limitations | ||
| # under the License. | ||
|
|
||
| from __future__ import annotations | ||
|
|
||
| import typing as tp | ||
|
|
||
| from gravitino.api.catalog import Catalog | ||
| from gravitino.api.catalog_change import CatalogChange | ||
| from gravitino.api.job.job_template import JobTemplate, JobType | ||
|
|
@@ -30,6 +34,7 @@ | |
| from gravitino.api.job.shell_job_template import ShellJobTemplate | ||
| from gravitino.api.job.spark_job_template import SparkJobTemplate | ||
| from gravitino.api.metalake_change import MetalakeChange | ||
| from gravitino.api.tag.tag_change import TagChange | ||
| from gravitino.client.fileset_catalog import FilesetCatalog | ||
| from gravitino.client.generic_model_catalog import GenericModelCatalog | ||
| from gravitino.client.relational_catalog import RelationalCatalog | ||
|
|
@@ -48,6 +53,7 @@ | |
| UpdateJobTemplateContentRequest, | ||
| ) | ||
| from gravitino.dto.requests.metalake_update_request import MetalakeUpdateRequest | ||
| from gravitino.dto.requests.tag_update_request import TagUpdateRequest | ||
| from gravitino.namespace import Namespace | ||
| from gravitino.utils import HTTPClient | ||
|
|
||
|
|
@@ -285,3 +291,44 @@ def to_job_template_update_request( | |
| return UpdateJobTemplateContentRequest(template_update_dto) | ||
|
|
||
| raise ValueError(f"Unknown change type: {type(change).__name__}") | ||
|
|
||
| @staticmethod | ||
| def to_tag_update_request( | ||
| change: TagChange, | ||
| ) -> tp.Union[ | ||
| TagUpdateRequest.RenameTagRequest, | ||
| TagUpdateRequest.UpdateTagCommentRequest, | ||
| TagUpdateRequest.SetTagPropertyRequest, | ||
| TagUpdateRequest.RemoveTagPropertyRequest, | ||
| ]: | ||
| """ | ||
| Convert the tag change to the corresponding tag update request. | ||
|
|
||
| Args: | ||
| change (TagChange): The tag change. | ||
|
|
||
| Raises: | ||
| ValueError: if the change type is not supported. | ||
|
|
||
| Returns: | ||
| tp.Union[ TagUpdateRequest.RenameTagRequest, | ||
| TagUpdateRequest.UpdateTagCommentRequest, | ||
| TagUpdateRequest.SetTagPropertyRequest, | ||
| TagUpdateRequest.RemoveTagPropertyRequest, | ||
| ]: The tag update request. | ||
| """ | ||
| if isinstance(change, TagChange.RenameTag): | ||
| return TagUpdateRequest.RenameTagRequest(change.new_name) | ||
|
|
||
| if isinstance(change, TagChange.UpdateTagComment): | ||
| return TagUpdateRequest.UpdateTagCommentRequest(change.new_comment) | ||
|
|
||
| if isinstance(change, TagChange.SetProperty): | ||
| return TagUpdateRequest.SetTagPropertyRequest( | ||
| change.name, | ||
| change.value, | ||
| ) | ||
| if isinstance(change, TagChange.RemoveProperty): | ||
| return TagUpdateRequest.RemoveTagPropertyRequest(change.removed_property) | ||
|
|
||
| raise ValueError(f"Unknown change type: {type(change)}") | ||
|
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. We should raise the similar exception to the Java client, |
||
| Original file line number | Diff line number | Diff line change | ||||||||||||||||||
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
|
|
@@ -41,6 +41,7 @@ | |||||||||||||||||||
| JobTemplateUpdatesRequest, | ||||||||||||||||||||
| ) | ||||||||||||||||||||
| from gravitino.dto.requests.tag_create_request import TagCreateRequest | ||||||||||||||||||||
| from gravitino.dto.requests.tag_updates_request import TagUpdatesRequest | ||||||||||||||||||||
| from gravitino.dto.responses.catalog_list_response import CatalogListResponse | ||||||||||||||||||||
| from gravitino.dto.responses.catalog_response import CatalogResponse | ||||||||||||||||||||
| from gravitino.dto.responses.drop_response import DropResponse | ||||||||||||||||||||
|
|
@@ -49,13 +50,18 @@ | |||||||||||||||||||
| from gravitino.dto.responses.job_response import JobResponse | ||||||||||||||||||||
| from gravitino.dto.responses.job_template_list_response import JobTemplateListResponse | ||||||||||||||||||||
| from gravitino.dto.responses.job_template_response import JobTemplateResponse | ||||||||||||||||||||
| from gravitino.dto.responses.tag_response import TagNamesListResponse, TagResponse | ||||||||||||||||||||
| from gravitino.dto.responses.tag_response import ( | ||||||||||||||||||||
| TagListResponse, | ||||||||||||||||||||
| TagNamesListResponse, | ||||||||||||||||||||
| TagResponse, | ||||||||||||||||||||
| ) | ||||||||||||||||||||
| from gravitino.exceptions.handlers.catalog_error_handler import CATALOG_ERROR_HANDLER | ||||||||||||||||||||
| from gravitino.exceptions.handlers.job_error_handler import JOB_ERROR_HANDLER | ||||||||||||||||||||
| from gravitino.exceptions.handlers.tag_error_handler import TAG_ERROR_HANDLER | ||||||||||||||||||||
| from gravitino.rest.rest_utils import encode_string | ||||||||||||||||||||
| from gravitino.utils.http_client import HTTPClient | ||||||||||||||||||||
| from gravitino.utils.precondition import Precondition | ||||||||||||||||||||
| from gravitino.utils.string_utils import StringUtils | ||||||||||||||||||||
|
|
||||||||||||||||||||
| logger = logging.getLogger(__name__) | ||||||||||||||||||||
|
|
||||||||||||||||||||
|
|
@@ -544,8 +550,20 @@ def list_tags_info(self) -> List[Tag]: | |||||||||||||||||||
| Raises: | ||||||||||||||||||||
| NoSuchMetalakeException: If the metalake does not exist. | ||||||||||||||||||||
| """ | ||||||||||||||||||||
| # TODO implement list_tags_info | ||||||||||||||||||||
| raise NotImplementedError() | ||||||||||||||||||||
| url = self.API_METALAKES_TAGS_PATH.format(encode_string(self.name())) | ||||||||||||||||||||
|
|
||||||||||||||||||||
| response = self.rest_client.get(url, error_handler=TAG_ERROR_HANDLER) | ||||||||||||||||||||
|
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I didn't see that parameters are sent with the API call similar to the Java client. gravitino/clients/client-java/src/main/java/org/apache/gravitino/client/GravitinoMetalake.java Lines 461 to 469 in 366f3ee
|
||||||||||||||||||||
| list_info_resp = TagListResponse.from_json(response.body, infer_missing=True) | ||||||||||||||||||||
| list_info_resp.validate() | ||||||||||||||||||||
|
|
||||||||||||||||||||
| return [ | ||||||||||||||||||||
| GenericTag( | ||||||||||||||||||||
| self.name, | ||||||||||||||||||||
| tag_dto, | ||||||||||||||||||||
| self.rest_client, | ||||||||||||||||||||
| ) | ||||||||||||||||||||
| for tag_dto in list_info_resp.tags() | ||||||||||||||||||||
| ] | ||||||||||||||||||||
|
|
||||||||||||||||||||
| def get_tag(self, tag_name) -> Tag: | ||||||||||||||||||||
| """ | ||||||||||||||||||||
|
|
@@ -623,8 +641,31 @@ def alter_tag(self, tag_name, *changes) -> Tag: | |||||||||||||||||||
| NoSuchTagException: If the tag does not exist. | ||||||||||||||||||||
| NoSuchMetalakeException: If the metalake does not exist. | ||||||||||||||||||||
| """ | ||||||||||||||||||||
| # TODO implement alter_tag | ||||||||||||||||||||
| raise NotImplementedError() | ||||||||||||||||||||
| Precondition.check_argument( | ||||||||||||||||||||
| StringUtils.is_not_blank(tag_name), | ||||||||||||||||||||
| "tag name must not be null or empty", | ||||||||||||||||||||
| ) | ||||||||||||||||||||
| updates = [DTOConverters.to_tag_update_request(change) for change in changes] | ||||||||||||||||||||
| update_req = TagUpdatesRequest(updates) | ||||||||||||||||||||
| update_req.validate() | ||||||||||||||||||||
|
|
||||||||||||||||||||
| url = self.API_METALAKES_TAG_PATH.format( | ||||||||||||||||||||
| encode_string(self.name()), encode_string(tag_name) | ||||||||||||||||||||
| ) | ||||||||||||||||||||
| response = self.rest_client.post( | ||||||||||||||||||||
| url, | ||||||||||||||||||||
| json=update_req, | ||||||||||||||||||||
| error_handler=TAG_ERROR_HANDLER, | ||||||||||||||||||||
| ) | ||||||||||||||||||||
| tag_resp: TagResponse = TagResponse.from_json(response.body, infer_missing=True) | ||||||||||||||||||||
| tag_resp.validate() | ||||||||||||||||||||
|
|
||||||||||||||||||||
| tag_resp.validate() | ||||||||||||||||||||
|
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Is this statement, |
||||||||||||||||||||
| return GenericTag( | ||||||||||||||||||||
| self.name(), | ||||||||||||||||||||
| tag_resp.tag(), | ||||||||||||||||||||
| self.rest_client, | ||||||||||||||||||||
| ) | ||||||||||||||||||||
|
|
||||||||||||||||||||
| def delete_tag(self, tag_name) -> bool: | ||||||||||||||||||||
| """ | ||||||||||||||||||||
|
|
@@ -640,5 +681,19 @@ def delete_tag(self, tag_name) -> bool: | |||||||||||||||||||
| NoSuchTagException: If the tag does not exist. | ||||||||||||||||||||
| NoSuchMetalakeException: If the metalake does not exist. | ||||||||||||||||||||
| """ | ||||||||||||||||||||
| # TODO implement delete_tag | ||||||||||||||||||||
| raise NotImplementedError() | ||||||||||||||||||||
| Precondition.check_argument( | ||||||||||||||||||||
| StringUtils.is_not_blank(tag_name), | ||||||||||||||||||||
| "tag name must not be null or empty", | ||||||||||||||||||||
| ) | ||||||||||||||||||||
|
|
||||||||||||||||||||
| url = self.API_METALAKES_TAG_PATH.format( | ||||||||||||||||||||
| encode_string(self.name()), encode_string(tag_name) | ||||||||||||||||||||
| ) | ||||||||||||||||||||
| response = self.rest_client.delete( | ||||||||||||||||||||
| url, | ||||||||||||||||||||
| error_handler=TAG_ERROR_HANDLER, | ||||||||||||||||||||
| ) | ||||||||||||||||||||
| drop_response = DropResponse.from_json(response.body, infer_missing=True) | ||||||||||||||||||||
| drop_response.validate() | ||||||||||||||||||||
|
|
||||||||||||||||||||
| return drop_response.dropped() | ||||||||||||||||||||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,38 @@ | ||
| # Licensed to the Apache Software Foundation (ASF) under one | ||
| # or more contributor license agreements. See the NOTICE file | ||
| # distributed with this work for additional information | ||
| # regarding copyright ownership. The ASF licenses this file | ||
| # to you under the Apache License, Version 2.0 (the | ||
| # "License"); you may not use this file except in compliance | ||
| # with the License. You may obtain a copy of the License at | ||
| # | ||
| # http://www.apache.org/licenses/LICENSE-2.0 | ||
| # | ||
| # Unless required by applicable law or agreed to in writing, | ||
| # software distributed under the License is distributed on an | ||
| # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY | ||
| # KIND, either express or implied. See the License for the | ||
| # specific language governing permissions and limitations | ||
| # under the License. | ||
|
|
||
|
|
||
| from __future__ import annotations | ||
|
|
||
| import random | ||
| import string | ||
|
|
||
|
|
||
| class RandomStringUtils: | ||
| @staticmethod | ||
| def random_string(length: int = 8) -> str: | ||
| """ | ||
| Generate a random string of specified length. | ||
|
|
||
| Args: | ||
| length (int, optional): The length of string. Defaults to 8. | ||
|
|
||
| Returns: | ||
| str: The random string. | ||
| """ | ||
| chars = string.ascii_letters + string.digits | ||
| return "".join(random.choices(chars, k=length)) |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,16 @@ | ||
| # Licensed to the Apache Software Foundation (ASF) under one | ||
| # or more contributor license agreements. See the NOTICE file | ||
| # distributed with this work for additional information | ||
| # regarding copyright ownership. The ASF licenses this file | ||
| # to you under the Apache License, Version 2.0 (the | ||
| # "License"); you may not use this file except in compliance | ||
| # with the License. You may obtain a copy of the License at | ||
| # | ||
| # http://www.apache.org/licenses/LICENSE-2.0 | ||
| # | ||
| # Unless required by applicable law or agreed to in writing, | ||
| # software distributed under the License is distributed on an | ||
| # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY | ||
| # KIND, either express or implied. See the License for the | ||
| # specific language governing permissions and limitations | ||
| # under the License. |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,99 @@ | ||
| # Licensed to the Apache Software Foundation (ASF) under one | ||
| # or more contributor license agreements. See the NOTICE file | ||
| # distributed with this work for additional information | ||
| # regarding copyright ownership. The ASF licenses this file | ||
| # to you under the Apache License, Version 2.0 (the | ||
| # "License"); you may not use this file except in compliance | ||
| # with the License. You may obtain a copy of the License at | ||
| # | ||
| # http://www.apache.org/licenses/LICENSE-2.0 | ||
| # | ||
| # Unless required by applicable law or agreed to in writing, | ||
| # software distributed under the License is distributed on an | ||
| # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY | ||
| # KIND, either express or implied. See the License for the | ||
| # specific language governing permissions and limitations | ||
| # under the License. | ||
|
|
||
|
|
||
| import json as _json | ||
| import unittest | ||
|
|
||
| from gravitino.api.tag.tag_change import TagChange | ||
| from gravitino.client.dto_converters import DTOConverters | ||
| from gravitino.dto.requests.tag_update_request import TagUpdateRequest | ||
|
|
||
|
|
||
| class TestDtoConverters(unittest.TestCase): | ||
| def test_to_tag_update_request_with_name(self) -> None: | ||
| new_tag_name = "new_tag_name" | ||
| change = TagChange.rename(new_tag_name) | ||
| rename_request = DTOConverters.to_tag_update_request(change) | ||
|
|
||
| self.assertTrue(isinstance(rename_request, TagUpdateRequest.RenameTagRequest)) | ||
| self.assertEqual(new_tag_name, rename_request.new_name) | ||
|
|
||
| json_str = _json.dumps( | ||
| { | ||
| "@type": "rename", | ||
| "newName": f"{new_tag_name}", | ||
| } | ||
| ) | ||
| self.assertEqual(json_str, rename_request.to_json()) | ||
|
|
||
| def test_to_tag_update_request_with_comment(self) -> None: | ||
| new_comment = "new_comment" | ||
| change = TagChange.update_comment(new_comment) | ||
| update_comment_request = DTOConverters.to_tag_update_request(change) | ||
|
|
||
| self.assertTrue( | ||
| isinstance(update_comment_request, TagUpdateRequest.UpdateTagCommentRequest) | ||
| ) | ||
|
|
||
| json_str = _json.dumps( | ||
| { | ||
| "@type": "updateComment", | ||
| "newComment": f"{new_comment}", | ||
| } | ||
| ) | ||
| self.assertEqual(json_str, update_comment_request.to_json()) | ||
|
|
||
| def test_to_tag_update_request_with_new_property(self) -> None: | ||
| new_prop = "key" | ||
| new_value = "value" | ||
| change = TagChange.set_property(new_prop, new_value) | ||
| set_property_request = DTOConverters.to_tag_update_request(change) | ||
|
|
||
| self.assertTrue( | ||
| isinstance(set_property_request, TagUpdateRequest.SetTagPropertyRequest) | ||
| ) | ||
| json_str = _json.dumps( | ||
| { | ||
| "@type": "setProperty", | ||
| "property": f"{new_prop}", | ||
| "value": f"{new_value}", | ||
| } | ||
| ) | ||
| self.assertEqual(json_str, set_property_request.to_json()) | ||
|
|
||
| def test_to_tag_update_request_with_remove_property(self) -> None: | ||
| removed_prop = "key" | ||
| change = TagChange.remove_property(removed_prop) | ||
| remove_property_request = DTOConverters.to_tag_update_request(change) | ||
|
|
||
| self.assertTrue( | ||
| isinstance( | ||
| remove_property_request, TagUpdateRequest.RemoveTagPropertyRequest | ||
| ) | ||
| ) | ||
| json_str = _json.dumps( | ||
| { | ||
| "@type": "removeProperty", | ||
| "property": f"{removed_prop}", | ||
| } | ||
| ) | ||
| self.assertEqual(json_str, remove_property_request.to_json()) | ||
|
|
||
| def test_to_tag_update_request_with_unsupport_type(self) -> None: | ||
| with self.assertRaises(ValueError): | ||
| DTOConverters.to_tag_update_request(None) |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
All of the four requests are subclasses of
TagUpdateRequestBase. I think we can define the return type as the base class.