diff --git a/pom.xml b/pom.xml index d576ddbb46..6ffbbc09ec 100644 --- a/pom.xml +++ b/pom.xml @@ -76,11 +76,12 @@ 2.3.0 1.15.0 32.0.1-jre - 3.4.2 - 1.19.4 + 3.5.0 + 2.46 1.5.4 3.0.0 4.13.2 + 5.9.3 1.8 1.0.0 3.1.0 @@ -323,14 +324,6 @@ commons-logging commons-logging-api - - com.sun.jersey - jersey-core - - - com.sun.jersey - jersey-server - org.eclipse.jdt core @@ -408,14 +401,6 @@ commons-logging commons-logging-api - - com.sun.jersey - jersey-core - - - com.sun.jersey - jersey-server - io.netty * @@ -768,6 +753,18 @@ ${junit.version} test + + org.junit.jupiter + junit-jupiter-api + ${junit.jupiter.version} + test + + + org.junit.vintage + junit-vintage-engine + ${junit.jupiter.version} + test + com.google.protobuf protobuf-java @@ -784,13 +781,13 @@ ${jettison.version} - com.sun.jersey + org.glassfish.jersey.core jersey-client ${jersey.version} - com.sun.jersey - jersey-json + org.glassfish.jersey.media + jersey-media-json-jackson ${jersey.version} diff --git a/tez-api/pom.xml b/tez-api/pom.xml index a2e449f1e6..a37bdf1706 100644 --- a/tez-api/pom.xml +++ b/tez-api/pom.xml @@ -90,12 +90,12 @@ junit - com.sun.jersey + org.glassfish.jersey.core jersey-client - com.sun.jersey - jersey-json + org.glassfish.jersey.media + jersey-media-json-jackson org.apache.hadoop diff --git a/tez-api/src/main/java/org/apache/tez/common/ATSConstants.java b/tez-api/src/main/java/org/apache/tez/common/ATSConstants.java index e3c90d3154..5f3d596eb0 100644 --- a/tez-api/src/main/java/org/apache/tez/common/ATSConstants.java +++ b/tez-api/src/main/java/org/apache/tez/common/ATSConstants.java @@ -27,15 +27,16 @@ public class ATSConstants { /* Top level keys */ public static final String ENTITIES = "entities"; - public static final String ENTITY = "entity"; - public static final String ENTITY_TYPE = "entitytype"; + public static final String ENTITY_ID = "entityId"; + public static final String ENTITY_TYPE = "entityType"; public static final String EVENTS = "events"; - public static final String EVENT_TYPE = "eventtype"; + public static final String EVENT_TYPE = "eventType"; public static final String TIMESTAMP = "ts"; - public static final String EVENT_INFO = "eventinfo"; + public static final String EVENT_INFO = "eventInfo"; public static final String RELATED_ENTITIES = "relatedEntities"; - public static final String PRIMARY_FILTERS = "primaryfilters"; - public static final String OTHER_INFO = "otherinfo"; + public static final String PRIMARY_FILTERS = "primaryFilters"; + public static final String OTHER_INFO = "otherInfo"; + public static final String DOMAIN_ID = "domainId"; /* Section for related entities */ public static final String APPLICATION_ID = "applicationId"; diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/client/DAGClientTimelineImpl.java b/tez-api/src/main/java/org/apache/tez/dag/api/client/DAGClientTimelineImpl.java index 46112cdf80..76e16b5cdd 100644 --- a/tez-api/src/main/java/org/apache/tez/dag/api/client/DAGClientTimelineImpl.java +++ b/tez-api/src/main/java/org/apache/tez/dag/api/client/DAGClientTimelineImpl.java @@ -29,6 +29,8 @@ import java.util.Set; import javax.annotation.Nullable; +import javax.ws.rs.client.Client; +import javax.ws.rs.client.WebTarget; import javax.ws.rs.core.MediaType; import org.apache.hadoop.classification.InterfaceAudience.Private; @@ -52,11 +54,6 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Joiner; -import com.sun.jersey.api.client.Client; -import com.sun.jersey.api.client.ClientHandlerException; -import com.sun.jersey.api.client.ClientResponse; -import com.sun.jersey.api.client.UniformInterfaceException; -import com.sun.jersey.api.client.WebResource; import org.codehaus.jettison.json.JSONArray; import org.codehaus.jettison.json.JSONException; @@ -212,7 +209,7 @@ public DAGStatus waitForCompletionWithStatusUpdates( @Override public void close() throws IOException { if (httpClient != null) { - httpClient.destroy(); + httpClient.close(); httpClient = null; } if (timelineReaderStrategy != null) { @@ -426,26 +423,14 @@ protected Map parseTaskStatsForVertexes() @VisibleForTesting protected JSONObject getJsonRootEntity(String url) throws TezException { try { - WebResource wr = getCachedHttpClient().resource(url); - ClientResponse response = wr.accept(MediaType.APPLICATION_JSON_TYPE) - .type(MediaType.APPLICATION_JSON_TYPE) - .get(ClientResponse.class); - - final ClientResponse.Status clientResponseStatus = response.getClientResponseStatus(); - if (clientResponseStatus != ClientResponse.Status.OK) { - throw new TezException("Failed to get response from YARN Timeline:" + - " errorCode:" + clientResponseStatus + ", url:" + url); - } - - return response.getEntity(JSONObject.class); - } catch (ClientHandlerException e) { + Client client = getCachedHttpClient(); + WebTarget target = client.target(url); + String json = target.request(MediaType.APPLICATION_JSON_TYPE) + .accept(MediaType.APPLICATION_JSON_TYPE) + .get(String.class); + return new JSONObject(json); + } catch (Exception e) { throw new TezException("Error processing response from YARN Timeline", e); - } catch (UniformInterfaceException e) { - throw new TezException("Error accessing content from YARN Timeline - unexpected response", e); - } catch (IllegalArgumentException e) { - throw new TezException("Error accessing content from YARN Timeline - invalid url", e); - } catch (IOException e) { - throw new TezException("Error failed to get http client", e); } } diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/client/TimelineReaderFactory.java b/tez-api/src/main/java/org/apache/tez/dag/api/client/TimelineReaderFactory.java index de1c157183..3611c45827 100644 --- a/tez-api/src/main/java/org/apache/tez/dag/api/client/TimelineReaderFactory.java +++ b/tez-api/src/main/java/org/apache/tez/dag/api/client/TimelineReaderFactory.java @@ -24,12 +24,13 @@ import java.lang.reflect.Method; import java.net.HttpURLConnection; import java.net.URL; -import java.net.URLEncoder; import java.security.GeneralSecurityException; import javax.net.ssl.HostnameVerifier; import javax.net.ssl.HttpsURLConnection; import javax.net.ssl.SSLSocketFactory; +import javax.ws.rs.client.Client; +import javax.ws.rs.client.ClientBuilder; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; @@ -42,13 +43,11 @@ import org.apache.tez.dag.api.TezException; import com.google.common.annotations.VisibleForTesting; -import com.sun.jersey.api.client.Client; -import com.sun.jersey.api.client.config.ClientConfig; -import com.sun.jersey.api.client.config.DefaultClientConfig; -import com.sun.jersey.client.urlconnection.HttpURLConnectionFactory; -import com.sun.jersey.client.urlconnection.URLConnectionClientHandler; -import com.sun.jersey.json.impl.provider.entity.JSONRootElementProvider; +import org.glassfish.jersey.client.ClientConfig; +import org.glassfish.jersey.client.HttpUrlConnectorProvider; +import org.glassfish.jersey.client.HttpUrlConnectorProvider.ConnectionFactory; +import org.glassfish.jersey.jackson.JacksonFeature; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -161,10 +160,7 @@ public TimelineReaderTokenAuthenticatedStrategy(final Configuration conf, @Override public Client getHttpClient() throws IOException { Authenticator authenticator; - UserGroupInformation ugi = UserGroupInformation.getCurrentUser(); - UserGroupInformation realUgi = ugi.getRealUser(); - String doAsUser; - ClientConfig clientConfig = new DefaultClientConfig(JSONRootElementProvider.App.class); + ClientConfig clientConfig = new ClientConfig().register(JacksonFeature.class); ConnectionConfigurator connectionConfigurator = getNewConnectionConf(useHttps, connTimeout, sslFactory); @@ -175,20 +171,18 @@ public Client getHttpClient() throws IOException { throw new IOException("Failed to get authenticator", e); } - if (realUgi != null) { - doAsUser = ugi.getShortUserName(); - } else { - doAsUser = null; - } + ConnectionFactory factory = url -> { + try { + return new AuthenticatedURL(authenticator) + .openConnection(url, new AuthenticatedURL.Token()); + } catch (Exception e) { + throw new IOException("Hadoop Authentication failed", e); + } + }; - HttpURLConnectionFactory connectionFactory; - try { - connectionFactory = new TokenAuthenticatedURLConnectionFactory(connectionConfigurator, authenticator, - doAsUser); - } catch (TezException e) { - throw new IOException("Fail to create TokenAuthenticatedURLConnectionFactory", e); - } - return new Client(new URLConnectionClientHandler(connectionFactory), clientConfig); + HttpUrlConnectorProvider connectorProvider = new HttpUrlConnectorProvider().connectionFactory(factory); + clientConfig.connectorProvider(connectorProvider); + return ClientBuilder.newClient(clientConfig); } private static Authenticator getTokenAuthenticator() throws TezException { @@ -203,42 +197,6 @@ private static Authenticator getTokenAuthenticator() throws TezException { return ReflectionUtils.createClazzInstance(authenticatorClazzName); } - private static class TokenAuthenticatedURLConnectionFactory implements HttpURLConnectionFactory { - - private final Authenticator authenticator; - private final ConnectionConfigurator connConfigurator; - private final String doAsUser; - private final AuthenticatedURL.Token token; - - public TokenAuthenticatedURLConnectionFactory(ConnectionConfigurator connConfigurator, - Authenticator authenticator, - String doAsUser) throws TezException { - this.connConfigurator = connConfigurator; - this.authenticator = authenticator; - this.doAsUser = doAsUser; - this.token = ReflectionUtils.createClazzInstance( - DELEGATION_TOKEN_AUTHENTICATED_URL_TOKEN_CLASS_NAME, null, null); - } - - @Override - public HttpURLConnection getHttpURLConnection(URL url) throws IOException { - try { - AuthenticatedURL authenticatedURL= ReflectionUtils.createClazzInstance( - DELEGATION_TOKEN_AUTHENTICATED_URL_CLAZZ_NAME, new Class[] { - delegationTokenAuthenticatorClazz, - ConnectionConfigurator.class - }, new Object[] { - authenticator, - connConfigurator - }); - return ReflectionUtils.invokeMethod(authenticatedURL, - delegationTokenAuthenticateURLOpenConnectionMethod, url, token, doAsUser); - } catch (Exception e) { - throw new IOException(e); - } - } - } - @Override public void close() { if (sslFactory != null) { @@ -265,31 +223,11 @@ public TimelineReaderPseudoAuthenticatedStrategy(final Configuration conf, @Override public Client getHttpClient() { - ClientConfig config = new DefaultClientConfig(JSONRootElementProvider.App.class); - HttpURLConnectionFactory urlFactory = new PseudoAuthenticatedURLConnectionFactory(connectionConf); - return new Client(new URLConnectionClientHandler(urlFactory), config); + ClientConfig config = new ClientConfig().register(JacksonFeature.class); + return ClientBuilder.newClient(config); } - @VisibleForTesting - protected static class PseudoAuthenticatedURLConnectionFactory implements HttpURLConnectionFactory { - private final ConnectionConfigurator connectionConf; - - public PseudoAuthenticatedURLConnectionFactory(ConnectionConfigurator connectionConf) { - this.connectionConf = connectionConf; - } - - @Override - public HttpURLConnection getHttpURLConnection(URL url) throws IOException { - String tokenString = (url.getQuery() == null ? "?" : "&") + "user.name=" + - URLEncoder.encode(UserGroupInformation.getCurrentUser().getShortUserName(), "UTF8"); - - HttpURLConnection httpURLConnection = - (HttpURLConnection) (new URL(url + tokenString)).openConnection(); - this.connectionConf.configure(httpURLConnection); - - return httpURLConnection; - } - } + // PseudoAuthenticatedURLConnectionFactory removed in Jersey 2 migration @Override public void close() { diff --git a/tez-api/src/test/java/org/apache/tez/dag/api/client/TestATSHttpClient.java b/tez-api/src/test/java/org/apache/tez/dag/api/client/TestATSHttpClient.java index 01edd801e2..d7aafac40e 100644 --- a/tez-api/src/test/java/org/apache/tez/dag/api/client/TestATSHttpClient.java +++ b/tez-api/src/test/java/org/apache/tez/dag/api/client/TestATSHttpClient.java @@ -95,7 +95,7 @@ public void testGetDagStatusSimple() throws TezException, JSONException, IOExcep final String jsonDagData = "{ " + - " otherinfo: { " + + " otherInfo: { " + " status: 'SUCCEEDED'," + " diagnostics: 'SAMPLE_DIAGNOSTICS'," + " counters: { counterGroups: [ " + @@ -108,9 +108,9 @@ public void testGetDagStatusSimple() throws TezException, JSONException, IOExcep "}"; final String jsonVertexData = "{entities:[ " + - "{otherinfo: {vertexName:'v1', numTasks:5,numFailedTasks:1,numSucceededTasks:2," + + "{otherInfo: {vertexName:'v1', numTasks:5,numFailedTasks:1,numSucceededTasks:2," + "numKilledTasks:3,numCompletedTasks:3}}," + - "{otherinfo: {vertexName:'v2',numTasks:10,numFailedTasks:1,numSucceededTasks:5," + + "{otherInfo: {vertexName:'v2',numTasks:10,numFailedTasks:1,numSucceededTasks:5," + "numKilledTasks:3,numCompletedTasks:4}}" + "]}"; @@ -151,7 +151,7 @@ public void testGetVertexStatusSimple() throws JSONException, TezException, IOEx Set statusOptions = new HashSet(1); statusOptions.add(StatusGetOpts.GET_COUNTERS); - final String jsonData = "{entities:[ {otherinfo:{numFailedTasks:1,numSucceededTasks:2," + + final String jsonData = "{entities:[ {otherInfo:{numFailedTasks:1,numSucceededTasks:2," + "status:'SUCCEEDED', vertexName:'vertex1name', numTasks:4, numKilledTasks: 3, " + "numCompletedTasks: 4, diagnostics: 'diagnostics1', " + "counters: { counterGroups: [ " + diff --git a/tez-api/src/test/java/org/apache/tez/dag/api/client/TestTimelineReaderFactory.java b/tez-api/src/test/java/org/apache/tez/dag/api/client/TestTimelineReaderFactory.java index e40e87e912..b7bb017c16 100644 --- a/tez-api/src/test/java/org/apache/tez/dag/api/client/TestTimelineReaderFactory.java +++ b/tez-api/src/test/java/org/apache/tez/dag/api/client/TestTimelineReaderFactory.java @@ -20,12 +20,7 @@ import static org.mockito.Mockito.mock; -import java.net.HttpURLConnection; -import java.net.URL; - import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.security.UserGroupInformation; -import org.apache.hadoop.security.authentication.client.ConnectionConfigurator; import org.apache.tez.dag.api.TezException; import org.apache.tez.dag.api.client.TimelineReaderFactory.TimelineReaderPseudoAuthenticatedStrategy; @@ -48,15 +43,11 @@ public void testShouldUseTokenDelegationAuthStrategyForHadoop26() throws TezExce } @Test(timeout = 5000) - public void testPseudoAuthenticatorConnectionUrlShouldHaveUserName() throws Exception { - ConnectionConfigurator connConf = mock(ConnectionConfigurator.class); - TimelineReaderPseudoAuthenticatedStrategy.PseudoAuthenticatedURLConnectionFactory - connectionFactory = new TimelineReaderPseudoAuthenticatedStrategy - .PseudoAuthenticatedURLConnectionFactory(connConf); - String inputUrl = "http://host:8080/path"; - String expectedUrl = inputUrl + "?user.name=" + UserGroupInformation.getCurrentUser().getShortUserName(); - HttpURLConnection httpURLConnection = connectionFactory.getHttpURLConnection(new URL(inputUrl)); - Assert.assertEquals(expectedUrl, httpURLConnection.getURL().toString()); + public void testPseudoStrategyCreatesJersey2Client() { + TimelineReaderPseudoAuthenticatedStrategy strategy = + new TimelineReaderPseudoAuthenticatedStrategy(new Configuration(), false, 1000); + Assert.assertNotNull(strategy.getHttpClient()); + strategy.close(); } } diff --git a/tez-dag/pom.xml b/tez-dag/pom.xml index 22f17a2364..b27e5eaf44 100644 --- a/tez-dag/pom.xml +++ b/tez-dag/pom.xml @@ -128,6 +128,11 @@ mockito-core test + + org.junit.jupiter + junit-jupiter-api + test + junit junit diff --git a/tez-dag/src/main/java/org/apache/tez/dag/history/logging/impl/HistoryEventJsonConversion.java b/tez-dag/src/main/java/org/apache/tez/dag/history/logging/impl/HistoryEventJsonConversion.java index c3a6dc0d65..8e841d7bd6 100644 --- a/tez-dag/src/main/java/org/apache/tez/dag/history/logging/impl/HistoryEventJsonConversion.java +++ b/tez-dag/src/main/java/org/apache/tez/dag/history/logging/impl/HistoryEventJsonConversion.java @@ -133,7 +133,7 @@ public static JSONObject convertToJson(HistoryEvent historyEvent) throws JSONExc private static JSONObject convertDAGRecoveredEvent(DAGRecoveredEvent event) throws JSONException { JSONObject jsonObject = new JSONObject(); - jsonObject.put(ATSConstants.ENTITY, + jsonObject.put(ATSConstants.ENTITY_ID, event.getDagID().toString()); jsonObject.put(ATSConstants.ENTITY_TYPE, EntityTypes.TEZ_DAG_ID.name()); @@ -168,7 +168,7 @@ private static JSONObject convertDAGRecoveredEvent(DAGRecoveredEvent event) private static JSONObject convertAppLaunchedEvent(AppLaunchedEvent event) throws JSONException { JSONObject jsonObject = new JSONObject(); - jsonObject.put(ATSConstants.ENTITY, + jsonObject.put(ATSConstants.ENTITY_ID, "tez_" + event.getApplicationId().toString()); jsonObject.put(ATSConstants.ENTITY_TYPE, EntityTypes.TEZ_APPLICATION.name()); @@ -186,7 +186,7 @@ private static JSONObject convertAppLaunchedEvent(AppLaunchedEvent event) throws private static JSONObject convertAMLaunchedEvent(AMLaunchedEvent event) throws JSONException { JSONObject jsonObject = new JSONObject(); - jsonObject.put(ATSConstants.ENTITY, + jsonObject.put(ATSConstants.ENTITY_ID, "tez_" + event.getApplicationAttemptId().toString()); jsonObject.put(ATSConstants.ENTITY_TYPE, EntityTypes.TEZ_APPLICATION_ATTEMPT.name()); @@ -194,12 +194,12 @@ private static JSONObject convertAMLaunchedEvent(AMLaunchedEvent event) throws J // Related Entities JSONArray relatedEntities = new JSONArray(); JSONObject appEntity = new JSONObject(); - appEntity.put(ATSConstants.ENTITY, + appEntity.put(ATSConstants.ENTITY_ID, event.getApplicationAttemptId().getApplicationId().toString()); appEntity.put(ATSConstants.ENTITY_TYPE, ATSConstants.APPLICATION_ID); JSONObject appAttemptEntity = new JSONObject(); - appAttemptEntity.put(ATSConstants.ENTITY, + appAttemptEntity.put(ATSConstants.ENTITY_ID, event.getApplicationAttemptId().toString()); appAttemptEntity.put(ATSConstants.ENTITY_TYPE, ATSConstants.APPLICATION_ATTEMPT_ID); @@ -227,7 +227,7 @@ private static JSONObject convertAMLaunchedEvent(AMLaunchedEvent event) throws J private static JSONObject convertAMStartedEvent(AMStartedEvent event) throws JSONException { JSONObject jsonObject = new JSONObject(); - jsonObject.put(ATSConstants.ENTITY, + jsonObject.put(ATSConstants.ENTITY_ID, "tez_" + event.getApplicationAttemptId().toString()); jsonObject.put(ATSConstants.ENTITY_TYPE, EntityTypes.TEZ_APPLICATION_ATTEMPT.name()); @@ -235,12 +235,12 @@ private static JSONObject convertAMStartedEvent(AMStartedEvent event) throws JSO // Related Entities JSONArray relatedEntities = new JSONArray(); JSONObject appEntity = new JSONObject(); - appEntity.put(ATSConstants.ENTITY, + appEntity.put(ATSConstants.ENTITY_ID, event.getApplicationAttemptId().getApplicationId().toString()); appEntity.put(ATSConstants.ENTITY_TYPE, ATSConstants.APPLICATION_ID); JSONObject appAttemptEntity = new JSONObject(); - appAttemptEntity.put(ATSConstants.ENTITY, + appAttemptEntity.put(ATSConstants.ENTITY_ID, event.getApplicationAttemptId().toString()); appAttemptEntity.put(ATSConstants.ENTITY_TYPE, ATSConstants.APPLICATION_ATTEMPT_ID); @@ -262,20 +262,20 @@ private static JSONObject convertAMStartedEvent(AMStartedEvent event) throws JSO private static JSONObject convertContainerLaunchedEvent(ContainerLaunchedEvent event) throws JSONException { JSONObject jsonObject = new JSONObject(); - jsonObject.put(ATSConstants.ENTITY, + jsonObject.put(ATSConstants.ENTITY_ID, "tez_" + event.getContainerId().toString()); jsonObject.put(ATSConstants.ENTITY_TYPE, EntityTypes.TEZ_CONTAINER_ID.name()); JSONArray relatedEntities = new JSONArray(); JSONObject appAttemptEntity = new JSONObject(); - appAttemptEntity.put(ATSConstants.ENTITY, + appAttemptEntity.put(ATSConstants.ENTITY_ID, event.getApplicationAttemptId().toString()); appAttemptEntity.put(ATSConstants.ENTITY_TYPE, EntityTypes.TEZ_APPLICATION_ATTEMPT.name()); JSONObject containerEntity = new JSONObject(); - containerEntity.put(ATSConstants.ENTITY, event.getContainerId().toString()); + containerEntity.put(ATSConstants.ENTITY_ID, event.getContainerId().toString()); containerEntity.put(ATSConstants.ENTITY_TYPE, ATSConstants.CONTAINER_ID); relatedEntities.put(appAttemptEntity); @@ -301,20 +301,20 @@ private static JSONObject convertContainerLaunchedEvent(ContainerLaunchedEvent e private static JSONObject convertContainerStoppedEvent(ContainerStoppedEvent event) throws JSONException { // structure is identical to ContainerLaunchedEvent JSONObject jsonObject = new JSONObject(); - jsonObject.put(ATSConstants.ENTITY, + jsonObject.put(ATSConstants.ENTITY_ID, "tez_" + event.getContainerId().toString()); jsonObject.put(ATSConstants.ENTITY_TYPE, EntityTypes.TEZ_CONTAINER_ID.name()); JSONArray relatedEntities = new JSONArray(); JSONObject appAttemptEntity = new JSONObject(); - appAttemptEntity.put(ATSConstants.ENTITY, + appAttemptEntity.put(ATSConstants.ENTITY_ID, event.getApplicationAttemptId().toString()); appAttemptEntity.put(ATSConstants.ENTITY_TYPE, EntityTypes.TEZ_APPLICATION_ATTEMPT.name()); JSONObject containerEntity = new JSONObject(); - containerEntity.put(ATSConstants.ENTITY, event.getContainerId().toString()); + containerEntity.put(ATSConstants.ENTITY_ID, event.getContainerId().toString()); containerEntity.put(ATSConstants.ENTITY_TYPE, ATSConstants.CONTAINER_ID); relatedEntities.put(appAttemptEntity); @@ -343,7 +343,7 @@ private static JSONObject convertContainerStoppedEvent(ContainerStoppedEvent eve private static JSONObject convertDAGFinishedEvent(DAGFinishedEvent event) throws JSONException { JSONObject jsonObject = new JSONObject(); - jsonObject.put(ATSConstants.ENTITY, + jsonObject.put(ATSConstants.ENTITY_ID, event.getDAGID().toString()); jsonObject.put(ATSConstants.ENTITY_TYPE, EntityTypes.TEZ_DAG_ID.name()); @@ -386,7 +386,7 @@ private static JSONObject convertDAGFinishedEvent(DAGFinishedEvent event) throws private static JSONObject convertDAGInitializedEvent(DAGInitializedEvent event) throws JSONException { JSONObject jsonObject = new JSONObject(); - jsonObject.put(ATSConstants.ENTITY, + jsonObject.put(ATSConstants.ENTITY_ID, event.getDAGID().toString()); jsonObject.put(ATSConstants.ENTITY_TYPE, EntityTypes.TEZ_DAG_ID.name()); @@ -418,7 +418,7 @@ private static JSONObject convertDAGInitializedEvent(DAGInitializedEvent event) private static JSONObject convertDAGStartedEvent(DAGStartedEvent event) throws JSONException { JSONObject jsonObject = new JSONObject(); - jsonObject.put(ATSConstants.ENTITY, + jsonObject.put(ATSConstants.ENTITY_ID, event.getDAGID().toString()); jsonObject.put(ATSConstants.ENTITY_TYPE, EntityTypes.TEZ_DAG_ID.name()); @@ -441,7 +441,7 @@ private static JSONObject convertDAGStartedEvent(DAGStartedEvent event) throws J private static JSONObject convertDAGSubmittedEvent(DAGSubmittedEvent event) throws JSONException { JSONObject jsonObject = new JSONObject(); - jsonObject.put(ATSConstants.ENTITY, + jsonObject.put(ATSConstants.ENTITY_ID, event.getDAGID().toString()); jsonObject.put(ATSConstants.ENTITY_TYPE, EntityTypes.TEZ_DAG_ID.name()); @@ -449,27 +449,27 @@ private static JSONObject convertDAGSubmittedEvent(DAGSubmittedEvent event) thro // Related Entities JSONArray relatedEntities = new JSONArray(); JSONObject tezAppEntity = new JSONObject(); - tezAppEntity.put(ATSConstants.ENTITY, + tezAppEntity.put(ATSConstants.ENTITY_ID, "tez_" + event.getApplicationAttemptId().getApplicationId().toString()); tezAppEntity.put(ATSConstants.ENTITY_TYPE, EntityTypes.TEZ_APPLICATION.name()); JSONObject tezAppAttemptEntity = new JSONObject(); - tezAppAttemptEntity.put(ATSConstants.ENTITY, + tezAppAttemptEntity.put(ATSConstants.ENTITY_ID, "tez_" + event.getApplicationAttemptId().toString()); tezAppAttemptEntity.put(ATSConstants.ENTITY_TYPE, EntityTypes.TEZ_APPLICATION_ATTEMPT.name()); JSONObject appEntity = new JSONObject(); - appEntity.put(ATSConstants.ENTITY, + appEntity.put(ATSConstants.ENTITY_ID, event.getApplicationAttemptId().getApplicationId().toString()); appEntity.put(ATSConstants.ENTITY_TYPE, ATSConstants.APPLICATION_ID); JSONObject appAttemptEntity = new JSONObject(); - appAttemptEntity.put(ATSConstants.ENTITY, + appAttemptEntity.put(ATSConstants.ENTITY_ID, event.getApplicationAttemptId().toString()); appAttemptEntity.put(ATSConstants.ENTITY_TYPE, ATSConstants.APPLICATION_ATTEMPT_ID); JSONObject userEntity = new JSONObject(); - userEntity.put(ATSConstants.ENTITY, + userEntity.put(ATSConstants.ENTITY_ID, event.getUser()); userEntity.put(ATSConstants.ENTITY_TYPE, ATSConstants.USER); @@ -531,7 +531,7 @@ private static JSONObject convertDAGSubmittedEvent(DAGSubmittedEvent event) thro private static JSONObject convertTaskAttemptFinishedEvent(TaskAttemptFinishedEvent event) throws JSONException { JSONObject jsonObject = new JSONObject(); - jsonObject.put(ATSConstants.ENTITY, event.getTaskAttemptID().toString()); + jsonObject.put(ATSConstants.ENTITY_ID, event.getTaskAttemptID().toString()); jsonObject.put(ATSConstants.ENTITY_TYPE, EntityTypes.TEZ_TASK_ATTEMPT_ID.name()); @@ -590,22 +590,22 @@ private static JSONObject convertTaskAttemptFinishedEvent(TaskAttemptFinishedEve private static JSONObject convertTaskAttemptStartedEvent(TaskAttemptStartedEvent event) throws JSONException { JSONObject jsonObject = new JSONObject(); - jsonObject.put(ATSConstants.ENTITY, event.getTaskAttemptID().toString()); + jsonObject.put(ATSConstants.ENTITY_ID, event.getTaskAttemptID().toString()); jsonObject.put(ATSConstants.ENTITY_TYPE, EntityTypes.TEZ_TASK_ATTEMPT_ID.name()); // Related entities JSONArray relatedEntities = new JSONArray(); JSONObject nodeEntity = new JSONObject(); - nodeEntity.put(ATSConstants.ENTITY, event.getNodeId().toString()); + nodeEntity.put(ATSConstants.ENTITY_ID, event.getNodeId().toString()); nodeEntity.put(ATSConstants.ENTITY_TYPE, ATSConstants.NODE_ID); JSONObject containerEntity = new JSONObject(); - containerEntity.put(ATSConstants.ENTITY, event.getContainerId().toString()); + containerEntity.put(ATSConstants.ENTITY_ID, event.getContainerId().toString()); containerEntity.put(ATSConstants.ENTITY_TYPE, ATSConstants.CONTAINER_ID); JSONObject taskEntity = new JSONObject(); - taskEntity.put(ATSConstants.ENTITY, event.getTaskID().toString()); + taskEntity.put(ATSConstants.ENTITY_ID, event.getTaskID().toString()); taskEntity.put(ATSConstants.ENTITY_TYPE, EntityTypes.TEZ_TASK_ID.name()); relatedEntities.put(nodeEntity); @@ -633,7 +633,7 @@ private static JSONObject convertTaskAttemptStartedEvent(TaskAttemptStartedEvent private static JSONObject convertTaskFinishedEvent(TaskFinishedEvent event) throws JSONException { JSONObject jsonObject = new JSONObject(); - jsonObject.put(ATSConstants.ENTITY, event.getTaskID().toString()); + jsonObject.put(ATSConstants.ENTITY_ID, event.getTaskID().toString()); jsonObject.put(ATSConstants.ENTITY_TYPE, EntityTypes.TEZ_TASK_ID.name()); // Events @@ -664,13 +664,13 @@ private static JSONObject convertTaskFinishedEvent(TaskFinishedEvent event) thro private static JSONObject convertTaskStartedEvent(TaskStartedEvent event) throws JSONException { JSONObject jsonObject = new JSONObject(); - jsonObject.put(ATSConstants.ENTITY, event.getTaskID().toString()); + jsonObject.put(ATSConstants.ENTITY_ID, event.getTaskID().toString()); jsonObject.put(ATSConstants.ENTITY_TYPE, EntityTypes.TEZ_TASK_ID.name()); // Related entities JSONArray relatedEntities = new JSONArray(); JSONObject vertexEntity = new JSONObject(); - vertexEntity.put(ATSConstants.ENTITY, event.getVertexID().toString()); + vertexEntity.put(ATSConstants.ENTITY_ID, event.getVertexID().toString()); vertexEntity.put(ATSConstants.ENTITY_TYPE, EntityTypes.TEZ_VERTEX_ID.name()); relatedEntities.put(vertexEntity); jsonObject.put(ATSConstants.RELATED_ENTITIES, relatedEntities); @@ -696,7 +696,7 @@ private static JSONObject convertTaskStartedEvent(TaskStartedEvent event) throws private static JSONObject convertVertexFinishedEvent(VertexFinishedEvent event) throws JSONException { JSONObject jsonObject = new JSONObject(); - jsonObject.put(ATSConstants.ENTITY, event.getVertexID().toString()); + jsonObject.put(ATSConstants.ENTITY_ID, event.getVertexID().toString()); jsonObject.put(ATSConstants.ENTITY_TYPE, EntityTypes.TEZ_VERTEX_ID.name()); // Events @@ -737,7 +737,7 @@ private static JSONObject convertVertexFinishedEvent(VertexFinishedEvent event) private static JSONObject convertVertexReconfigureDoneEvent(VertexConfigurationDoneEvent event) throws JSONException { JSONObject jsonObject = new JSONObject(); - jsonObject.put(ATSConstants.ENTITY, event.getVertexID().toString()); + jsonObject.put(ATSConstants.ENTITY_ID, event.getVertexID().toString()); jsonObject.put(ATSConstants.ENTITY_TYPE, EntityTypes.TEZ_VERTEX_ID.name()); // Events @@ -772,13 +772,13 @@ private static JSONObject convertVertexReconfigureDoneEvent(VertexConfigurationD private static JSONObject convertVertexInitializedEvent(VertexInitializedEvent event) throws JSONException { JSONObject jsonObject = new JSONObject(); - jsonObject.put(ATSConstants.ENTITY, event.getVertexID().toString()); + jsonObject.put(ATSConstants.ENTITY_ID, event.getVertexID().toString()); jsonObject.put(ATSConstants.ENTITY_TYPE, EntityTypes.TEZ_VERTEX_ID.name()); // Related entities JSONArray relatedEntities = new JSONArray(); JSONObject vertexEntity = new JSONObject(); - vertexEntity.put(ATSConstants.ENTITY, event.getDAGID().toString()); + vertexEntity.put(ATSConstants.ENTITY_ID, event.getDAGID().toString()); vertexEntity.put(ATSConstants.ENTITY_TYPE, EntityTypes.TEZ_DAG_ID.name()); relatedEntities.put(vertexEntity); jsonObject.put(ATSConstants.RELATED_ENTITIES, relatedEntities); @@ -812,13 +812,13 @@ private static JSONObject convertVertexInitializedEvent(VertexInitializedEvent e private static JSONObject convertVertexStartedEvent(VertexStartedEvent event) throws JSONException { JSONObject jsonObject = new JSONObject(); - jsonObject.put(ATSConstants.ENTITY, event.getVertexID().toString()); + jsonObject.put(ATSConstants.ENTITY_ID, event.getVertexID().toString()); jsonObject.put(ATSConstants.ENTITY_TYPE, EntityTypes.TEZ_VERTEX_ID.name()); // Related entities JSONArray relatedEntities = new JSONArray(); JSONObject vertexEntity = new JSONObject(); - vertexEntity.put(ATSConstants.ENTITY, event.getDAGID().toString()); + vertexEntity.put(ATSConstants.ENTITY_ID, event.getDAGID().toString()); vertexEntity.put(ATSConstants.ENTITY_TYPE, EntityTypes.TEZ_DAG_ID.name()); relatedEntities.put(vertexEntity); jsonObject.put(ATSConstants.RELATED_ENTITIES, relatedEntities); diff --git a/tez-dag/src/test/java/org/apache/tez/dag/history/logging/impl/TestHistoryEventJsonConversion.java b/tez-dag/src/test/java/org/apache/tez/dag/history/logging/impl/TestHistoryEventJsonConversion.java index ae95aca331..9852cbfcb9 100644 --- a/tez-dag/src/test/java/org/apache/tez/dag/history/logging/impl/TestHistoryEventJsonConversion.java +++ b/tez-dag/src/test/java/org/apache/tez/dag/history/logging/impl/TestHistoryEventJsonConversion.java @@ -231,7 +231,7 @@ public void testConvertVertexReconfigureDoneEvent() throws JSONException { JSONObject jsonObject = HistoryEventJsonConversion.convertToJson(event); Assert.assertNotNull(jsonObject); - Assert.assertEquals(vId.toString(), jsonObject.getString(ATSConstants.ENTITY)); + Assert.assertEquals(vId.toString(), jsonObject.getString(ATSConstants.ENTITY_ID)); Assert.assertEquals(ATSConstants.TEZ_VERTEX_ID, jsonObject.get(ATSConstants.ENTITY_TYPE)); JSONArray events = jsonObject.getJSONArray(ATSConstants.EVENTS); diff --git a/tez-ext-service-tests/pom.xml b/tez-ext-service-tests/pom.xml index ef1d8c29d1..5fc41427b6 100644 --- a/tez-ext-service-tests/pom.xml +++ b/tez-ext-service-tests/pom.xml @@ -53,6 +53,11 @@ hadoop-hdfs test + + org.junit.jupiter + junit-jupiter-api + test + junit junit diff --git a/tez-plugins/tez-aux-services/pom.xml b/tez-plugins/tez-aux-services/pom.xml index d935d8ef74..ccda3270cc 100644 --- a/tez-plugins/tez-aux-services/pom.xml +++ b/tez-plugins/tez-aux-services/pom.xml @@ -127,6 +127,11 @@ mockito-core test + + org.junit.jupiter + junit-jupiter-api + test + org.apache.hadoop hadoop-hdfs diff --git a/tez-plugins/tez-history-parser/pom.xml b/tez-plugins/tez-history-parser/pom.xml index 4bb312eee9..17502c55d8 100644 --- a/tez-plugins/tez-history-parser/pom.xml +++ b/tez-plugins/tez-history-parser/pom.xml @@ -145,8 +145,19 @@ test - com.sun.jersey - jersey-json + org.junit.jupiter + junit-jupiter-api + test + + + org.junit.vintage + junit-vintage-engine + test + + + org.glassfish.jersey.media + jersey-media-json-jackson + test org.apache.hadoop diff --git a/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/ATSImportTool.java b/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/ATSImportTool.java index 94e0449929..455e92e9b7 100644 --- a/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/ATSImportTool.java +++ b/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/ATSImportTool.java @@ -23,17 +23,18 @@ import java.io.File; import java.io.FileOutputStream; import java.io.IOException; -import java.net.HttpURLConnection; import java.net.URI; import java.net.URISyntaxException; -import java.net.URL; -import java.net.URLEncoder; import java.util.Iterator; import java.util.Objects; import java.util.zip.ZipEntry; import java.util.zip.ZipOutputStream; +import javax.ws.rs.client.Client; +import javax.ws.rs.client.ClientBuilder; +import javax.ws.rs.client.WebTarget; import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.DefaultParser; @@ -42,11 +43,9 @@ import org.apache.commons.cli.Options; import org.apache.commons.cli.ParseException; import org.apache.commons.io.IOUtils; -import org.apache.commons.io.LineIterator; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configured; import org.apache.hadoop.http.HttpConfig; -import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.util.Tool; import org.apache.hadoop.util.ToolRunner; import org.apache.hadoop.yarn.conf.YarnConfiguration; @@ -60,16 +59,6 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Joiner; import com.google.common.base.Strings; -import com.sun.jersey.api.client.Client; -import com.sun.jersey.api.client.ClientHandlerException; -import com.sun.jersey.api.client.ClientResponse; -import com.sun.jersey.api.client.UniformInterfaceException; -import com.sun.jersey.api.client.WebResource; -import com.sun.jersey.api.client.config.ClientConfig; -import com.sun.jersey.api.client.config.DefaultClientConfig; -import com.sun.jersey.client.urlconnection.HttpURLConnectionFactory; -import com.sun.jersey.client.urlconnection.URLConnectionClientHandler; -import com.sun.jersey.json.impl.provider.entity.JSONRootElementProvider; import org.codehaus.jettison.json.JSONArray; import org.codehaus.jettison.json.JSONException; @@ -169,7 +158,7 @@ private void download() throws Exception { throw e; } finally { if (httpClient != null) { - httpClient.destroy(); + httpClient.close(); } IOUtils.closeQuietly(fos); } @@ -279,76 +268,56 @@ private void downloadJSONArrayFromATS(String url, ZipOutputStream zos, String ta //Set the last item in entities as the fromId url = baseUrl + "&fromId=" - + entities.getJSONObject(entities.length() - 1).getString(Constants.ENTITY); + + entities.getJSONObject(entities.length() - 1).getString(Constants.ENTITY_ID); - String firstItem = entities.getJSONObject(0).getString(Constants.ENTITY); - String lastItem = entities.getJSONObject(entities.length() - 1).getString(Constants.ENTITY); + String firstItem = entities.getJSONObject(0).getString(Constants.ENTITY_ID); + String lastItem = entities.getJSONObject(entities.length() - 1).getString(Constants.ENTITY_ID); LOG.info("Downloaded={}, First item={}, LastItem={}, new url={}", downloadedCount, firstItem, lastItem, url); } } - private void logErrorMessage(ClientResponse response) throws IOException { - LOG.error("Response status={}", response.getClientResponseStatus().toString()); - LineIterator it = null; + private void logErrorMessage(Response response) { + LOG.error("Response status={}", Integer.toString(response.getStatus())); try { - it = IOUtils.lineIterator(response.getEntityInputStream(), UTF8); - while (it.hasNext()) { - String line = it.nextLine(); - LOG.error(line); - } - } finally { - if (it != null) { - it.close(); + String entity = response.readEntity(String.class); + if (entity != null) { + LOG.error(entity); } + } catch (Exception ignore) { + // ignore } } //For secure cluster, this should work as long as valid ticket is available in the node. private JSONObject getJsonRootEntity(String url) throws TezException, IOException { try { - WebResource wr = getHttpClient().resource(url); - ClientResponse response = wr.accept(MediaType.APPLICATION_JSON_TYPE) - .type(MediaType.APPLICATION_JSON_TYPE) - .get(ClientResponse.class); + WebTarget target = getHttpClient().target(url); + Response response = target.request(MediaType.APPLICATION_JSON_TYPE) + .accept(MediaType.APPLICATION_JSON_TYPE) + .get(); - if (response.getClientResponseStatus() != ClientResponse.Status.OK) { + if (response.getStatus() != Response.Status.OK.getStatusCode()) { // In the case of secure cluster, if there is any auth exception it sends the data back as // a html page and JSON parsing could throw exceptions. Instead, get the stream contents // completely and log it in case of error. logErrorMessage(response); throw new TezException("Failed to get response from YARN Timeline: url: " + url); } - return response.getEntity(JSONObject.class); - } catch (ClientHandlerException e) { + String json = response.readEntity(String.class); + return new JSONObject(json); + } catch (Exception e) { throw new TezException("Error processing response from YARN Timeline. URL=" + url, e); - } catch (UniformInterfaceException e) { - throw new TezException("Error accessing content from YARN Timeline - unexpected response. " - + "URL=" + url, e); - } catch (IllegalArgumentException e) { - throw new TezException("Error accessing content from YARN Timeline - invalid url. URL=" + url, - e); } } private Client getHttpClient() { if (httpClient == null) { - ClientConfig config = new DefaultClientConfig(JSONRootElementProvider.App.class); - HttpURLConnectionFactory urlFactory = new PseudoAuthenticatedURLConnectionFactory(); - return new Client(new URLConnectionClientHandler(urlFactory), config); + return ClientBuilder.newClient(); } return httpClient; } - static class PseudoAuthenticatedURLConnectionFactory implements HttpURLConnectionFactory { - @Override - public HttpURLConnection getHttpURLConnection(URL url) throws IOException { - String tokenString = (url.getQuery() == null ? "?" : "&") + "user.name=" + - URLEncoder.encode(UserGroupInformation.getCurrentUser().getShortUserName(), "UTF8"); - return (HttpURLConnection) (new URL(url.toString() + tokenString)).openConnection(); - } - } - @Override public int run(String[] args) throws Exception { try { diff --git a/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/SimpleHistoryParser.java b/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/SimpleHistoryParser.java index 3ec6fabcbc..ef3fc3a0cd 100644 --- a/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/SimpleHistoryParser.java +++ b/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/SimpleHistoryParser.java @@ -179,7 +179,7 @@ protected void postProcessMaps(Map vertexJsonMap, if (relatedEntities == null) { //This can happen when CONTAINER_EXITED abruptly. (e.g Container failed, exitCode=1) LOG.debug("entity {} did not have related entities", - jsonObject.optJSONObject(Constants.ENTITY)); + jsonObject.optJSONObject(Constants.ENTITY_ID)); } else { JSONObject subJsonObject = relatedEntities.optJSONObject(0); if (subJsonObject != null) { @@ -187,7 +187,7 @@ protected void postProcessMaps(Map vertexJsonMap, if (!Strings.isNullOrEmpty(nodeId) && nodeId.equalsIgnoreCase(Constants.NODE_ID)) { //populate it in otherInfo JSONObject otherInfo = jsonObject.optJSONObject(Constants.OTHER_INFO); - String nodeIdVal = subJsonObject.optString(Constants.ENTITY); + String nodeIdVal = subJsonObject.optString(Constants.ENTITY_ID); if (otherInfo != null && nodeIdVal != null) { otherInfo.put(Constants.NODE_ID, nodeIdVal); } @@ -201,7 +201,7 @@ protected void postProcessMaps(Map vertexJsonMap, .equalsIgnoreCase(Constants.CONTAINER_ID)) { //populate it in otherInfo JSONObject otherInfo = jsonObject.optJSONObject(Constants.OTHER_INFO); - String containerIdVal = subJsonObject.optString(Constants.ENTITY); + String containerIdVal = subJsonObject.optString(Constants.ENTITY_ID); if (otherInfo != null && containerIdVal != null) { otherInfo.put(Constants.CONTAINER_ID, containerIdVal); } @@ -224,7 +224,7 @@ protected void readEventsFromSource(String dagId, JSONObjectSource source, while (source.hasNext()) { JSONObject jsonObject = source.next(); - String entity = jsonObject.getString(Constants.ENTITY); + String entity = jsonObject.getString(Constants.ENTITY_ID); String entityType = jsonObject.getString(Constants.ENTITY_TYPE); switch (entityType) { case Constants.TEZ_DAG_ID: @@ -254,7 +254,7 @@ protected void readEventsFromSource(String dagId, JSONObjectSource source, JSONObject subEntity = relatedEntities.getJSONObject(i); String subEntityType = subEntity.optString(Constants.ENTITY_TYPE); if (subEntityType != null && subEntityType.equals(Constants.USER)) { - userName = subEntity.getString(Constants.ENTITY); + userName = subEntity.getString(Constants.ENTITY_ID); break; } } diff --git a/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/datamodel/DagInfo.java b/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/datamodel/DagInfo.java index 923214cdf9..29ddac49f4 100644 --- a/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/datamodel/DagInfo.java +++ b/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/datamodel/DagInfo.java @@ -102,7 +102,7 @@ public class DagInfo extends BaseInfo { Preconditions.checkArgument(jsonObject.getString(Constants.ENTITY_TYPE).equalsIgnoreCase (Constants.TEZ_DAG_ID)); - dagId = StringInterner.intern(jsonObject.getString(Constants.ENTITY)); + dagId = StringInterner.intern(jsonObject.getString(Constants.ENTITY_ID)); //Parse additional Info JSONObject otherInfoNode = jsonObject.getJSONObject(Constants.OTHER_INFO); diff --git a/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/datamodel/TaskAttemptInfo.java b/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/datamodel/TaskAttemptInfo.java index d5b845dcf7..9c52958029 100644 --- a/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/datamodel/TaskAttemptInfo.java +++ b/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/datamodel/TaskAttemptInfo.java @@ -98,7 +98,7 @@ public String getTaskAttemptId() { jsonObject.getString(Constants.ENTITY_TYPE).equalsIgnoreCase (Constants.TEZ_TASK_ATTEMPT_ID)); - taskAttemptId = StringInterner.intern(jsonObject.optString(Constants.ENTITY)); + taskAttemptId = StringInterner.intern(jsonObject.optString(Constants.ENTITY_ID)); //Parse additional Info final JSONObject otherInfoNode = jsonObject.getJSONObject(Constants.OTHER_INFO); diff --git a/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/datamodel/TaskInfo.java b/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/datamodel/TaskInfo.java index 2a1c929785..20e8d45583 100644 --- a/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/datamodel/TaskInfo.java +++ b/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/datamodel/TaskInfo.java @@ -73,7 +73,7 @@ public class TaskInfo extends BaseInfo { jsonObject.getString(Constants.ENTITY_TYPE).equalsIgnoreCase (Constants.TEZ_TASK_ID)); - taskId = StringInterner.intern(jsonObject.optString(Constants.ENTITY)); + taskId = StringInterner.intern(jsonObject.optString(Constants.ENTITY_ID)); //Parse additional Info final JSONObject otherInfoNode = jsonObject.getJSONObject(Constants.OTHER_INFO); diff --git a/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/datamodel/VertexInfo.java b/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/datamodel/VertexInfo.java index 72ecf44183..b91c1155a3 100644 --- a/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/datamodel/VertexInfo.java +++ b/tez-plugins/tez-history-parser/src/main/java/org/apache/tez/history/parser/datamodel/VertexInfo.java @@ -94,7 +94,7 @@ public class VertexInfo extends BaseInfo { jsonObject.getString(Constants.ENTITY_TYPE).equalsIgnoreCase (Constants.TEZ_VERTEX_ID)); - vertexId = StringInterner.intern(jsonObject.optString(Constants.ENTITY)); + vertexId = StringInterner.intern(jsonObject.optString(Constants.ENTITY_ID)); taskInfoMap = Maps.newHashMap(); inEdgeList = Lists.newLinkedList(); diff --git a/tez-plugins/tez-protobuf-history-plugin/src/main/java/org/apache/tez/dag/history/logging/proto/HistoryEventProtoJsonConversion.java b/tez-plugins/tez-protobuf-history-plugin/src/main/java/org/apache/tez/dag/history/logging/proto/HistoryEventProtoJsonConversion.java index 01292a2fbd..5b51bda8c4 100644 --- a/tez-plugins/tez-protobuf-history-plugin/src/main/java/org/apache/tez/dag/history/logging/proto/HistoryEventProtoJsonConversion.java +++ b/tez-plugins/tez-protobuf-history-plugin/src/main/java/org/apache/tez/dag/history/logging/proto/HistoryEventProtoJsonConversion.java @@ -115,7 +115,7 @@ public static JSONObject convertToJson(HistoryEventProto historyEvent) throws JS private static JSONObject convertDAGRecoveredEvent(HistoryEventProto event) throws JSONException { JSONObject jsonObject = new JSONObject(); - jsonObject.put(ATSConstants.ENTITY, event.getDagId()); + jsonObject.put(ATSConstants.ENTITY_ID, event.getDagId()); jsonObject.put(ATSConstants.ENTITY_TYPE, EntityTypes.TEZ_DAG_ID.name()); // Related Entities not needed as should have been done in @@ -142,7 +142,7 @@ private static JSONObject convertDAGRecoveredEvent(HistoryEventProto event) thro private static JSONObject convertAppLaunchedEvent(HistoryEventProto event) throws JSONException { JSONObject jsonObject = new JSONObject(); - jsonObject.put(ATSConstants.ENTITY, "tez_" + event.getAppId().toString()); + jsonObject.put(ATSConstants.ENTITY_ID, "tez_" + event.getAppId().toString()); jsonObject.put(ATSConstants.ENTITY_TYPE, EntityTypes.TEZ_APPLICATION.name()); // Other info to tag with Tez App @@ -157,16 +157,16 @@ private static JSONObject convertAppLaunchedEvent(HistoryEventProto event) throw private static JSONObject convertAMLaunchedEvent(HistoryEventProto event) throws JSONException { JSONObject jsonObject = new JSONObject(); - jsonObject.put(ATSConstants.ENTITY, "tez_" + event.getAppAttemptId().toString()); + jsonObject.put(ATSConstants.ENTITY_ID, "tez_" + event.getAppAttemptId().toString()); jsonObject.put(ATSConstants.ENTITY_TYPE, EntityTypes.TEZ_APPLICATION_ATTEMPT.name()); // Related Entities JSONArray relatedEntities = new JSONArray(); JSONObject appEntity = new JSONObject(); - appEntity.put(ATSConstants.ENTITY, event.getAppId().toString()); + appEntity.put(ATSConstants.ENTITY_ID, event.getAppId().toString()); appEntity.put(ATSConstants.ENTITY_TYPE, ATSConstants.APPLICATION_ID); JSONObject appAttemptEntity = new JSONObject(); - appAttemptEntity.put(ATSConstants.ENTITY, event.getAppAttemptId().toString()); + appAttemptEntity.put(ATSConstants.ENTITY_ID, event.getAppAttemptId().toString()); appAttemptEntity.put(ATSConstants.ENTITY_TYPE, ATSConstants.APPLICATION_ATTEMPT_ID); relatedEntities.put(appEntity); relatedEntities.put(appAttemptEntity); @@ -192,16 +192,16 @@ private static JSONObject convertAMLaunchedEvent(HistoryEventProto event) throws private static JSONObject convertAMStartedEvent(HistoryEventProto event) throws JSONException { JSONObject jsonObject = new JSONObject(); - jsonObject.put(ATSConstants.ENTITY, "tez_" + event.getAppAttemptId().toString()); + jsonObject.put(ATSConstants.ENTITY_ID, "tez_" + event.getAppAttemptId().toString()); jsonObject.put(ATSConstants.ENTITY_TYPE, EntityTypes.TEZ_APPLICATION_ATTEMPT.name()); // Related Entities JSONArray relatedEntities = new JSONArray(); JSONObject appEntity = new JSONObject(); - appEntity.put(ATSConstants.ENTITY, event.getAppId().toString()); + appEntity.put(ATSConstants.ENTITY_ID, event.getAppId().toString()); appEntity.put(ATSConstants.ENTITY_TYPE, ATSConstants.APPLICATION_ID); JSONObject appAttemptEntity = new JSONObject(); - appAttemptEntity.put(ATSConstants.ENTITY, event.getAppAttemptId().toString()); + appAttemptEntity.put(ATSConstants.ENTITY_ID, event.getAppAttemptId().toString()); appAttemptEntity.put(ATSConstants.ENTITY_TYPE, ATSConstants.APPLICATION_ATTEMPT_ID); relatedEntities.put(appEntity); relatedEntities.put(appAttemptEntity); @@ -222,17 +222,17 @@ private static JSONObject convertAMStartedEvent(HistoryEventProto event) throws private static JSONObject convertContainerLaunchedEvent(HistoryEventProto event) throws JSONException { JSONObject jsonObject = new JSONObject(); - jsonObject.put(ATSConstants.ENTITY, + jsonObject.put(ATSConstants.ENTITY_ID, "tez_" + getDataValueByKey(event, ATSConstants.CONTAINER_ID)); jsonObject.put(ATSConstants.ENTITY_TYPE, EntityTypes.TEZ_CONTAINER_ID.name()); JSONArray relatedEntities = new JSONArray(); JSONObject appAttemptEntity = new JSONObject(); - appAttemptEntity.put(ATSConstants.ENTITY, event.getAppAttemptId().toString()); + appAttemptEntity.put(ATSConstants.ENTITY_ID, event.getAppAttemptId().toString()); appAttemptEntity.put(ATSConstants.ENTITY_TYPE, EntityTypes.TEZ_APPLICATION_ATTEMPT.name()); JSONObject containerEntity = new JSONObject(); - containerEntity.put(ATSConstants.ENTITY, getDataValueByKey(event, ATSConstants.CONTAINER_ID)); + containerEntity.put(ATSConstants.ENTITY_ID, getDataValueByKey(event, ATSConstants.CONTAINER_ID)); containerEntity.put(ATSConstants.ENTITY_TYPE, ATSConstants.CONTAINER_ID); relatedEntities.put(appAttemptEntity); @@ -258,17 +258,17 @@ private static JSONObject convertContainerStoppedEvent(HistoryEventProto event) throws JSONException { // structure is identical to ContainerLaunchedEvent JSONObject jsonObject = new JSONObject(); - jsonObject.put(ATSConstants.ENTITY, + jsonObject.put(ATSConstants.ENTITY_ID, "tez_" + getDataValueByKey(event, ATSConstants.CONTAINER_ID)); jsonObject.put(ATSConstants.ENTITY_TYPE, EntityTypes.TEZ_CONTAINER_ID.name()); JSONArray relatedEntities = new JSONArray(); JSONObject appAttemptEntity = new JSONObject(); - appAttemptEntity.put(ATSConstants.ENTITY, event.getAppAttemptId().toString()); + appAttemptEntity.put(ATSConstants.ENTITY_ID, event.getAppAttemptId().toString()); appAttemptEntity.put(ATSConstants.ENTITY_TYPE, EntityTypes.TEZ_APPLICATION_ATTEMPT.name()); JSONObject containerEntity = new JSONObject(); - containerEntity.put(ATSConstants.ENTITY, getDataValueByKey(event, ATSConstants.CONTAINER_ID)); + containerEntity.put(ATSConstants.ENTITY_ID, getDataValueByKey(event, ATSConstants.CONTAINER_ID)); containerEntity.put(ATSConstants.ENTITY_TYPE, ATSConstants.CONTAINER_ID); relatedEntities.put(appAttemptEntity); @@ -297,7 +297,7 @@ private static JSONObject convertContainerStoppedEvent(HistoryEventProto event) private static JSONObject convertDAGFinishedEvent(HistoryEventProto event) throws JSONException { JSONObject jsonObject = new JSONObject(); - jsonObject.put(ATSConstants.ENTITY, event.getDagId()); + jsonObject.put(ATSConstants.ENTITY_ID, event.getDagId()); jsonObject.put(ATSConstants.ENTITY_TYPE, EntityTypes.TEZ_DAG_ID.name()); // Related Entities not needed as should have been done in @@ -341,7 +341,7 @@ private static JSONObject convertDAGFinishedEvent(HistoryEventProto event) throw private static JSONObject convertDAGInitializedEvent(HistoryEventProto event) throws JSONException { JSONObject jsonObject = new JSONObject(); - jsonObject.put(ATSConstants.ENTITY, event.getDagId()); + jsonObject.put(ATSConstants.ENTITY_ID, event.getDagId()); jsonObject.put(ATSConstants.ENTITY_TYPE, EntityTypes.TEZ_DAG_ID.name()); // Related Entities not needed as should have been done in @@ -364,7 +364,7 @@ private static JSONObject convertDAGInitializedEvent(HistoryEventProto event) private static JSONObject convertDAGStartedEvent(HistoryEventProto event) throws JSONException { JSONObject jsonObject = new JSONObject(); - jsonObject.put(ATSConstants.ENTITY, event.getDagId()); + jsonObject.put(ATSConstants.ENTITY_ID, event.getDagId()); jsonObject.put(ATSConstants.ENTITY_TYPE, EntityTypes.TEZ_DAG_ID.name()); // Related Entities not needed as should have been done in @@ -384,25 +384,25 @@ private static JSONObject convertDAGStartedEvent(HistoryEventProto event) throws private static JSONObject convertDAGSubmittedEvent(HistoryEventProto event) throws JSONException { JSONObject jsonObject = new JSONObject(); - jsonObject.put(ATSConstants.ENTITY, event.getDagId()); + jsonObject.put(ATSConstants.ENTITY_ID, event.getDagId()); jsonObject.put(ATSConstants.ENTITY_TYPE, EntityTypes.TEZ_DAG_ID.name()); // Related Entities JSONArray relatedEntities = new JSONArray(); JSONObject tezAppEntity = new JSONObject(); - tezAppEntity.put(ATSConstants.ENTITY, "tez_" + event.getAppId().toString()); + tezAppEntity.put(ATSConstants.ENTITY_ID, "tez_" + event.getAppId().toString()); tezAppEntity.put(ATSConstants.ENTITY_TYPE, EntityTypes.TEZ_APPLICATION.name()); JSONObject tezAppAttemptEntity = new JSONObject(); - tezAppAttemptEntity.put(ATSConstants.ENTITY, "tez_" + event.getAppAttemptId().toString()); + tezAppAttemptEntity.put(ATSConstants.ENTITY_ID, "tez_" + event.getAppAttemptId().toString()); tezAppAttemptEntity.put(ATSConstants.ENTITY_TYPE, EntityTypes.TEZ_APPLICATION_ATTEMPT.name()); JSONObject appEntity = new JSONObject(); - appEntity.put(ATSConstants.ENTITY, event.getAppId().toString()); + appEntity.put(ATSConstants.ENTITY_ID, event.getAppId().toString()); appEntity.put(ATSConstants.ENTITY_TYPE, ATSConstants.APPLICATION_ID); JSONObject appAttemptEntity = new JSONObject(); - appAttemptEntity.put(ATSConstants.ENTITY, event.getAppAttemptId().toString()); + appAttemptEntity.put(ATSConstants.ENTITY_ID, event.getAppAttemptId().toString()); appAttemptEntity.put(ATSConstants.ENTITY_TYPE, ATSConstants.APPLICATION_ATTEMPT_ID); JSONObject userEntity = new JSONObject(); - userEntity.put(ATSConstants.ENTITY, event.getUser()); + userEntity.put(ATSConstants.ENTITY_ID, event.getUser()); userEntity.put(ATSConstants.ENTITY_TYPE, ATSConstants.USER); relatedEntities.put(tezAppEntity); @@ -452,7 +452,7 @@ private static JSONObject convertDAGSubmittedEvent(HistoryEventProto event) thro private static JSONObject convertTaskAttemptFinishedEvent(HistoryEventProto event) throws JSONException { JSONObject jsonObject = new JSONObject(); - jsonObject.put(ATSConstants.ENTITY, event.getTaskAttemptId()); + jsonObject.put(ATSConstants.ENTITY_ID, event.getTaskAttemptId()); jsonObject.put(ATSConstants.ENTITY_TYPE, EntityTypes.TEZ_TASK_ATTEMPT_ID.name()); // Events @@ -503,21 +503,21 @@ private static JSONObject convertTaskAttemptFinishedEvent(HistoryEventProto even private static JSONObject convertTaskAttemptStartedEvent(HistoryEventProto event) throws JSONException { JSONObject jsonObject = new JSONObject(); - jsonObject.put(ATSConstants.ENTITY, event.getTaskAttemptId()); + jsonObject.put(ATSConstants.ENTITY_ID, event.getTaskAttemptId()); jsonObject.put(ATSConstants.ENTITY_TYPE, EntityTypes.TEZ_TASK_ATTEMPT_ID.name()); // Related entities JSONArray relatedEntities = new JSONArray(); JSONObject nodeEntity = new JSONObject(); - nodeEntity.put(ATSConstants.ENTITY, getDataValueByKey(event, ATSConstants.NODE_ID)); + nodeEntity.put(ATSConstants.ENTITY_ID, getDataValueByKey(event, ATSConstants.NODE_ID)); nodeEntity.put(ATSConstants.ENTITY_TYPE, ATSConstants.NODE_ID); JSONObject containerEntity = new JSONObject(); - containerEntity.put(ATSConstants.ENTITY, getDataValueByKey(event, ATSConstants.CONTAINER_ID)); + containerEntity.put(ATSConstants.ENTITY_ID, getDataValueByKey(event, ATSConstants.CONTAINER_ID)); containerEntity.put(ATSConstants.ENTITY_TYPE, ATSConstants.CONTAINER_ID); JSONObject taskEntity = new JSONObject(); - taskEntity.put(ATSConstants.ENTITY, event.getTaskAttemptId()); + taskEntity.put(ATSConstants.ENTITY_ID, event.getTaskAttemptId()); taskEntity.put(ATSConstants.ENTITY_TYPE, EntityTypes.TEZ_TASK_ID.name()); relatedEntities.put(nodeEntity); @@ -546,7 +546,7 @@ private static JSONObject convertTaskAttemptStartedEvent(HistoryEventProto event private static JSONObject convertTaskFinishedEvent(HistoryEventProto event) throws JSONException { JSONObject jsonObject = new JSONObject(); - jsonObject.put(ATSConstants.ENTITY, event.getTaskId()); + jsonObject.put(ATSConstants.ENTITY_ID, event.getTaskId()); jsonObject.put(ATSConstants.ENTITY_TYPE, EntityTypes.TEZ_TASK_ID.name()); // Events @@ -577,13 +577,13 @@ private static JSONObject convertTaskFinishedEvent(HistoryEventProto event) thro private static JSONObject convertTaskStartedEvent(HistoryEventProto event) throws JSONException { JSONObject jsonObject = new JSONObject(); - jsonObject.put(ATSConstants.ENTITY, event.getTaskId()); + jsonObject.put(ATSConstants.ENTITY_ID, event.getTaskId()); jsonObject.put(ATSConstants.ENTITY_TYPE, EntityTypes.TEZ_TASK_ID.name()); // Related entities JSONArray relatedEntities = new JSONArray(); JSONObject vertexEntity = new JSONObject(); - vertexEntity.put(ATSConstants.ENTITY, event.getVertexId()); + vertexEntity.put(ATSConstants.ENTITY_ID, event.getVertexId()); vertexEntity.put(ATSConstants.ENTITY_TYPE, EntityTypes.TEZ_VERTEX_ID.name()); relatedEntities.put(vertexEntity); jsonObject.put(ATSConstants.RELATED_ENTITIES, relatedEntities); @@ -610,7 +610,7 @@ private static JSONObject convertTaskStartedEvent(HistoryEventProto event) throw private static JSONObject convertVertexFinishedEvent(HistoryEventProto event) throws JSONException { JSONObject jsonObject = new JSONObject(); - jsonObject.put(ATSConstants.ENTITY, event.getVertexId()); + jsonObject.put(ATSConstants.ENTITY_ID, event.getVertexId()); jsonObject.put(ATSConstants.ENTITY_TYPE, EntityTypes.TEZ_VERTEX_ID.name()); // Events @@ -653,7 +653,7 @@ private static JSONObject convertVertexFinishedEvent(HistoryEventProto event) private static JSONObject convertVertexReconfigureDoneEvent(HistoryEventProto event) throws JSONException { JSONObject jsonObject = new JSONObject(); - jsonObject.put(ATSConstants.ENTITY, event.getVertexId()); + jsonObject.put(ATSConstants.ENTITY_ID, event.getVertexId()); jsonObject.put(ATSConstants.ENTITY_TYPE, EntityTypes.TEZ_VERTEX_ID.name()); // Events @@ -681,13 +681,13 @@ private static JSONObject convertVertexReconfigureDoneEvent(HistoryEventProto ev private static JSONObject convertVertexInitializedEvent(HistoryEventProto event) throws JSONException { JSONObject jsonObject = new JSONObject(); - jsonObject.put(ATSConstants.ENTITY, event.getVertexId()); + jsonObject.put(ATSConstants.ENTITY_ID, event.getVertexId()); jsonObject.put(ATSConstants.ENTITY_TYPE, EntityTypes.TEZ_VERTEX_ID.name()); // Related entities JSONArray relatedEntities = new JSONArray(); JSONObject vertexEntity = new JSONObject(); - vertexEntity.put(ATSConstants.ENTITY, event.getDagId()); + vertexEntity.put(ATSConstants.ENTITY_ID, event.getDagId()); vertexEntity.put(ATSConstants.ENTITY_TYPE, EntityTypes.TEZ_DAG_ID.name()); relatedEntities.put(vertexEntity); jsonObject.put(ATSConstants.RELATED_ENTITIES, relatedEntities); @@ -720,13 +720,13 @@ private static JSONObject convertVertexInitializedEvent(HistoryEventProto event) private static JSONObject convertVertexStartedEvent(HistoryEventProto event) throws JSONException { JSONObject jsonObject = new JSONObject(); - jsonObject.put(ATSConstants.ENTITY, event.getVertexId()); + jsonObject.put(ATSConstants.ENTITY_ID, event.getVertexId()); jsonObject.put(ATSConstants.ENTITY_TYPE, EntityTypes.TEZ_VERTEX_ID.name()); // Related entities JSONArray relatedEntities = new JSONArray(); JSONObject vertexEntity = new JSONObject(); - vertexEntity.put(ATSConstants.ENTITY, event.getDagId()); + vertexEntity.put(ATSConstants.ENTITY_ID, event.getDagId()); vertexEntity.put(ATSConstants.ENTITY_TYPE, EntityTypes.TEZ_DAG_ID.name()); relatedEntities.put(vertexEntity); jsonObject.put(ATSConstants.RELATED_ENTITIES, relatedEntities); diff --git a/tez-plugins/tez-yarn-timeline-history-with-acls/pom.xml b/tez-plugins/tez-yarn-timeline-history-with-acls/pom.xml index dc8ecc1cb4..819e543519 100644 --- a/tez-plugins/tez-yarn-timeline-history-with-acls/pom.xml +++ b/tez-plugins/tez-yarn-timeline-history-with-acls/pom.xml @@ -135,8 +135,18 @@ test - com.sun.jersey - jersey-json + org.junit.jupiter + junit-jupiter-api + test + + + org.junit.vintage + junit-vintage-engine + test + + + org.glassfish.jersey.media + jersey-media-json-jackson test diff --git a/tez-plugins/tez-yarn-timeline-history-with-acls/src/test/java/org/apache/tez/dag/history/ats/acls/TestATSHistoryWithACLs.java b/tez-plugins/tez-yarn-timeline-history-with-acls/src/test/java/org/apache/tez/dag/history/ats/acls/TestATSHistoryWithACLs.java index 5ec3ff3b93..afd5d77ef7 100644 --- a/tez-plugins/tez-yarn-timeline-history-with-acls/src/test/java/org/apache/tez/dag/history/ats/acls/TestATSHistoryWithACLs.java +++ b/tez-plugins/tez-yarn-timeline-history-with-acls/src/test/java/org/apache/tez/dag/history/ats/acls/TestATSHistoryWithACLs.java @@ -31,7 +31,11 @@ import java.util.List; import java.util.Random; +import javax.ws.rs.client.Client; +import javax.ws.rs.client.ClientBuilder; +import javax.ws.rs.client.WebTarget; import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; @@ -46,6 +50,7 @@ import org.apache.hadoop.yarn.api.records.timeline.TimelineEvent; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.tez.client.TezClient; +import org.apache.tez.common.ATSConstants; import org.apache.tez.common.ReflectionUtils; import org.apache.tez.common.security.DAGAccessControls; import org.apache.tez.dag.api.DAG; @@ -66,9 +71,6 @@ import org.apache.tez.tests.MiniTezClusterWithTimeline; import com.google.common.collect.Sets; -import com.sun.jersey.api.client.Client; -import com.sun.jersey.api.client.ClientResponse; -import com.sun.jersey.api.client.WebResource; import org.codehaus.jettison.json.JSONArray; import org.codehaus.jettison.json.JSONException; @@ -148,23 +150,25 @@ public static void tearDown() throws InterruptedException { // To be replaced after Timeline has java APIs for domains private K getTimelineData(String url, Class clazz) { - Client client = new Client(); - WebResource resource = client.resource(url); - - ClientResponse response = resource.accept(MediaType.APPLICATION_JSON) - .get(ClientResponse.class); + Client client = ClientBuilder.newClient(); + WebTarget target = client.target(url); + Response response = target.request(MediaType.APPLICATION_JSON).get(); assertEquals(200, response.getStatus()); - assertTrue(MediaType.APPLICATION_JSON_TYPE.isCompatible(response.getType())); - - JSONObject entity = response.getEntity(JSONObject.class); - K converted = null; + assertTrue(MediaType.APPLICATION_JSON_TYPE.isCompatible(response.getMediaType())); + String entityStr = response.readEntity(String.class); try { - converted = convertJSONObjectToTimelineObject(entity, clazz); + JSONObject jsonObject = new JSONObject(entityStr); + // Handle the nesting introduced by Jersey 2/Jackson + JSONObject effectiveJson = jsonObject.has("domain") ? jsonObject.getJSONObject("domain") : jsonObject; + K converted = convertJSONObjectToTimelineObject(effectiveJson, clazz); + assertNotNull(converted); + return converted; } catch (JSONException e) { throw new RuntimeException(e); + } finally { + response.close(); + client.close(); } - assertNotNull(converted); - return converted; } private K convertJSONObjectToTimelineObject(JSONObject jsonObj, Class clazz) throws JSONException { @@ -178,9 +182,9 @@ private K convertJSONObjectToTimelineObject(JSONObject jsonObj, Class cla return (K) domain; } else if (clazz == TimelineEntity.class) { TimelineEntity entity = new TimelineEntity(); - entity.setEntityId(jsonObj.getString("entity")); - entity.setEntityType(jsonObj.getString("entitytype")); - entity.setDomainId(jsonObj.getString("domain")); + entity.setEntityId(jsonObj.getString(ATSConstants.ENTITY_ID)); + entity.setEntityType(jsonObj.getString(ATSConstants.ENTITY_TYPE)); + entity.setDomainId(jsonObj.getString(ATSConstants.DOMAIN_ID)); entity.setEvents(getEventsFromJSON(jsonObj)); return (K) entity; } else { @@ -194,7 +198,7 @@ private List getEventsFromJSON(JSONObject jsonObj) throws JSONExc JSONArray arrEvents = jsonObj.getJSONArray("events"); for (int i = 0; i < arrEvents.length(); i++) { TimelineEvent event = new TimelineEvent(); - event.setEventType(((JSONObject) arrEvents.get(i)).getString("eventtype")); + event.setEventType(((JSONObject) arrEvents.get(i)).getString(ATSConstants.EVENT_TYPE)); events.add(event); } return events; @@ -453,12 +457,12 @@ public void testDagLoggingDisabled() throws Exception { historyLoggingService.handle(new DAGHistoryEvent(tezDAGID, submittedEvent)); Thread.sleep(1000l); String url = "http://" + timelineAddress + "/ws/v1/timeline/TEZ_DAG_ID/"+event.getDAGID(); - Client client = new Client(); - WebResource resource = client.resource(url); - - ClientResponse response = resource.accept(MediaType.APPLICATION_JSON) - .get(ClientResponse.class); + Client client = ClientBuilder.newClient(); + WebTarget target = client.target(url); + Response response = target.request(MediaType.APPLICATION_JSON).get(); assertEquals(404, response.getStatus()); + response.close(); + client.close(); } /** @@ -498,17 +502,18 @@ public void testDagLoggingEnabled() throws Exception { historyLoggingService.handle(new DAGHistoryEvent(tezDAGID, submittedEvent)); Thread.sleep(1000l); String url = "http://" + timelineAddress + "/ws/v1/timeline/TEZ_DAG_ID/"+event.getDAGID(); - Client client = new Client(); - WebResource resource = client.resource(url); - - ClientResponse response = resource.accept(MediaType.APPLICATION_JSON) - .get(ClientResponse.class); + Client client = ClientBuilder.newClient(); + WebTarget target = client.target(url); + Response response = target.request(MediaType.APPLICATION_JSON).get(); assertEquals(200, response.getStatus()); - assertTrue(MediaType.APPLICATION_JSON_TYPE.isCompatible(response.getType())); - JSONObject entityJson = response.getEntity(JSONObject.class); + assertTrue(MediaType.APPLICATION_JSON_TYPE.isCompatible(response.getMediaType())); + String entityStr = response.readEntity(String.class); + JSONObject entityJson = new JSONObject(entityStr); TimelineEntity entity = convertJSONObjectToTimelineObject(entityJson, TimelineEntity.class); assertEquals(entity.getEntityType(), "TEZ_DAG_ID"); assertEquals(entity.getEvents().get(0).getEventType(), HistoryEventType.DAG_SUBMITTED.toString()); + response.close(); + client.close(); } private static final String atsHistoryACLManagerClassName = diff --git a/tez-plugins/tez-yarn-timeline-history-with-fs/pom.xml b/tez-plugins/tez-yarn-timeline-history-with-fs/pom.xml index 19930ccda0..227c681a3f 100644 --- a/tez-plugins/tez-yarn-timeline-history-with-fs/pom.xml +++ b/tez-plugins/tez-yarn-timeline-history-with-fs/pom.xml @@ -50,6 +50,11 @@ test-jar test + + org.junit.jupiter + junit-jupiter-api + test + org.apache.tez tez-tests @@ -143,8 +148,8 @@ test - com.sun.jersey - jersey-json + org.glassfish.jersey.media + jersey-media-json-jackson test diff --git a/tez-plugins/tez-yarn-timeline-history/pom.xml b/tez-plugins/tez-yarn-timeline-history/pom.xml index 1be2c44bfd..dcc8c01894 100644 --- a/tez-plugins/tez-yarn-timeline-history/pom.xml +++ b/tez-plugins/tez-yarn-timeline-history/pom.xml @@ -96,6 +96,11 @@ hadoop-mapreduce-client-common test + + org.junit.jupiter + junit-jupiter-api + test + org.apache.hadoop hadoop-mapreduce-client-jobclient @@ -129,8 +134,8 @@ test - com.sun.jersey - jersey-json + org.glassfish.jersey.media + jersey-media-json-jackson test diff --git a/tez-plugins/tez-yarn-timeline-history/src/test/java/org/apache/tez/dag/history/logging/ats/TestATSHistoryWithMiniCluster.java b/tez-plugins/tez-yarn-timeline-history/src/test/java/org/apache/tez/dag/history/logging/ats/TestATSHistoryWithMiniCluster.java index 451f281c5b..938a6f610b 100644 --- a/tez-plugins/tez-yarn-timeline-history/src/test/java/org/apache/tez/dag/history/logging/ats/TestATSHistoryWithMiniCluster.java +++ b/tez-plugins/tez-yarn-timeline-history/src/test/java/org/apache/tez/dag/history/logging/ats/TestATSHistoryWithMiniCluster.java @@ -21,7 +21,11 @@ import java.io.IOException; import java.util.Random; +import javax.ws.rs.client.Client; +import javax.ws.rs.client.ClientBuilder; +import javax.ws.rs.client.WebTarget; import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; @@ -40,10 +44,6 @@ import org.apache.tez.runtime.library.processor.SleepProcessor.SleepProcessorConfig; import org.apache.tez.tests.MiniTezClusterWithTimeline; -import com.sun.jersey.api.client.Client; -import com.sun.jersey.api.client.ClientResponse; -import com.sun.jersey.api.client.WebResource; - import org.junit.AfterClass; import org.junit.Assert; import org.junit.BeforeClass; @@ -115,16 +115,18 @@ public static void tearDown() throws InterruptedException { // To be replaced after Timeline has java APIs for domains private K getTimelineData(String url, Class clazz) { - Client client = new Client(); - WebResource resource = client.resource(url); + Client client = ClientBuilder.newClient(); + WebTarget target = client.target(url); - ClientResponse response = resource.accept(MediaType.APPLICATION_JSON) - .get(ClientResponse.class); + Response response = target.request(MediaType.APPLICATION_JSON) + .get(); Assert.assertEquals(200, response.getStatus()); - Assert.assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + Assert.assertTrue(MediaType.APPLICATION_JSON_TYPE.isCompatible(response.getMediaType())); - K entity = response.getEntity(clazz); + K entity = response.readEntity(clazz); Assert.assertNotNull(entity); + response.close(); + client.close(); return entity; } diff --git a/tez-tests/pom.xml b/tez-tests/pom.xml index 46016d383f..47324841ce 100644 --- a/tez-tests/pom.xml +++ b/tez-tests/pom.xml @@ -71,6 +71,11 @@ hadoop-hdfs test + + org.junit.jupiter + junit-jupiter-api + test + org.apache.hadoop hadoop-yarn-api diff --git a/tez-tools/analyzers/job-analyzer/pom.xml b/tez-tools/analyzers/job-analyzer/pom.xml index e797842e6c..e9d199aadb 100644 --- a/tez-tools/analyzers/job-analyzer/pom.xml +++ b/tez-tools/analyzers/job-analyzer/pom.xml @@ -153,8 +153,19 @@ test - com.sun.jersey - jersey-json + org.junit.jupiter + junit-jupiter-api + test + + + org.junit.vintage + junit-vintage-engine + test + + + org.glassfish.jersey.media + jersey-media-json-jackson + test org.apache.hadoop diff --git a/tez-tools/analyzers/job-analyzer/src/main/java/org/apache/tez/analyzer/utils/Utils.java b/tez-tools/analyzers/job-analyzer/src/main/java/org/apache/tez/analyzer/utils/Utils.java index 4fb8ebb354..00aeba88f6 100644 --- a/tez-tools/analyzers/job-analyzer/src/main/java/org/apache/tez/analyzer/utils/Utils.java +++ b/tez-tools/analyzers/job-analyzer/src/main/java/org/apache/tez/analyzer/utils/Utils.java @@ -23,14 +23,14 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; +import javax.annotation.Nullable; + import org.apache.tez.dag.utils.Graph; import org.apache.tez.history.parser.datamodel.AdditionalInputOutputDetails; import org.apache.tez.history.parser.datamodel.DagInfo; import org.apache.tez.history.parser.datamodel.EdgeInfo; import org.apache.tez.history.parser.datamodel.VertexInfo; -import com.sun.istack.Nullable; - public final class Utils { private static Pattern sanitizeLabelPattern = Pattern.compile("[:\\-\\W]+");