Skip to content

TEZ-4688 TEZ-4648: Tez upgrade to Hadoop 3.5.0 and jersey 2.x #474

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
maheshrajus wants to merge 7 commits into
base: master
Choose a base branch
from
Open

TEZ-4688 TEZ-4648: Tez upgrade to Hadoop 3.5.0 and jersey 2.x #474

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
b28d2f7
TEZ-4688: Tez upgrade to Hadoop 3.5.0
maheshrajus Apr 8, 2026
fc9865d
TEZ-4648: Upgrade jersey to 2.x
abstractdog Sep 15, 2025
fcd930a
jersey2 migration
abstractdog Sep 15, 2025
a1e9c45
jupiter fixes
abstractdog Sep 16, 2025
30c08bf
spotbugs fix
maheshrajus Apr 8, 2026
8f159f9
test failures fix
maheshrajus Apr 6, 2026
76a32cb
review comments fix
maheshrajus Apr 22, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
39 changes: 18 additions & 21 deletions pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -76,11 +76,12 @@
<plexus-velocity.version>2.3.0</plexus-velocity.version>
<frontend-maven-plugin.version>1.15.0</frontend-maven-plugin.version>
<guava.version>32.0.1-jre</guava.version>
<hadoop.version>3.4.2</hadoop.version>
<jersey.version>1.19.4</jersey.version>
<hadoop.version>3.5.0</hadoop.version>
Comment thread
maheshrajus marked this conversation as resolved.
<jersey.version>2.46</jersey.version>
<jettison.version>1.5.4</jettison.version>
<jsr305.version>3.0.0</jsr305.version>
<junit.version>4.13.2</junit.version>
<junit.jupiter.version>5.9.3</junit.jupiter.version>
Comment thread
maheshrajus marked this conversation as resolved.
<leveldbjni-all.version>1.8</leveldbjni-all.version>
<lifecycle-mapping.version>1.0.0</lifecycle-mapping.version>
<metrics-core.version>3.1.0</metrics-core.version>
Expand Down Expand Up @@ -323,14 +324,6 @@
<groupId>commons-logging</groupId>
<artifactId>commons-logging-api</artifactId>
</exclusion>
<exclusion>
<groupId>com.sun.jersey</groupId>
<artifactId>jersey-core</artifactId>
</exclusion>
<exclusion>
<groupId>com.sun.jersey</groupId>
<artifactId>jersey-server</artifactId>
</exclusion>
<exclusion>
<groupId>org.eclipse.jdt</groupId>
<artifactId>core</artifactId>
Expand Down Expand Up @@ -408,14 +401,6 @@
<groupId>commons-logging</groupId>
<artifactId>commons-logging-api</artifactId>
</exclusion>
<exclusion>
<groupId>com.sun.jersey</groupId>
<artifactId>jersey-core</artifactId>
</exclusion>
<exclusion>
<groupId>com.sun.jersey</groupId>
<artifactId>jersey-server</artifactId>
</exclusion>
<exclusion>
<groupId>io.netty</groupId>
<artifactId>*</artifactId>
Expand Down Expand Up @@ -768,6 +753,18 @@
<version>${junit.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.junit.jupiter</groupId>
<artifactId>junit-jupiter-api</artifactId>
<version>${junit.jupiter.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.junit.vintage</groupId>
<artifactId>junit-vintage-engine</artifactId>
<version>${junit.jupiter.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>com.google.protobuf</groupId>
<artifactId>protobuf-java</artifactId>
Expand All @@ -784,13 +781,13 @@
<version>${jettison.version}</version>
</dependency>
<dependency>
<groupId>com.sun.jersey</groupId>
<groupId>org.glassfish.jersey.core</groupId>
<artifactId>jersey-client</artifactId>
<version>${jersey.version}</version>
</dependency>
<dependency>
<groupId>com.sun.jersey</groupId>
<artifactId>jersey-json</artifactId>
<groupId>org.glassfish.jersey.media</groupId>
<artifactId>jersey-media-json-jackson</artifactId>
<version>${jersey.version}</version>
</dependency>
<!-- BouncyCastle should be in default scope to make their way to tez.tar.gz since Hadoop 3.4.1 -->
Expand Down
6 changes: 3 additions & 3 deletions tez-api/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -90,12 +90,12 @@
<artifactId>junit</artifactId>
</dependency>
<dependency>
<groupId>com.sun.jersey</groupId>
<groupId>org.glassfish.jersey.core</groupId>
<artifactId>jersey-client</artifactId>
</dependency>
<dependency>
<groupId>com.sun.jersey</groupId>
<artifactId>jersey-json</artifactId>
<groupId>org.glassfish.jersey.media</groupId>
<artifactId>jersey-media-json-jackson</artifactId>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
Expand Down
13 changes: 7 additions & 6 deletions tez-api/src/main/java/org/apache/tez/common/ATSConstants.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,15 +27,16 @@ public class ATSConstants {

/* Top level keys */
public static final String ENTITIES = "entities";
public static final String ENTITY = "entity";
public static final String ENTITY_TYPE = "entitytype";
public static final String ENTITY_ID = "entityId";
public static final String ENTITY_TYPE = "entityType";
public static final String EVENTS = "events";
public static final String EVENT_TYPE = "eventtype";
public static final String EVENT_TYPE = "eventType";
public static final String TIMESTAMP = "ts";
public static final String EVENT_INFO = "eventinfo";
public static final String EVENT_INFO = "eventInfo";
public static final String RELATED_ENTITIES = "relatedEntities";
public static final String PRIMARY_FILTERS = "primaryfilters";
public static final String OTHER_INFO = "otherinfo";
public static final String PRIMARY_FILTERS = "primaryFilters";
public static final String OTHER_INFO = "otherInfo";
public static final String DOMAIN_ID = "domainId";

/* Section for related entities */
public static final String APPLICATION_ID = "applicationId";
Expand Down
35 changes: 10 additions & 25 deletions tez-api/src/main/java/org/apache/tez/dag/api/client/DAGClientTimelineImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,8 @@
import java.util.Set;

import javax.annotation.Nullable;
import javax.ws.rs.client.Client;
import javax.ws.rs.client.WebTarget;
import javax.ws.rs.core.MediaType;

import org.apache.hadoop.classification.InterfaceAudience.Private;
Expand All @@ -52,11 +54,6 @@

import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Joiner;
import com.sun.jersey.api.client.Client;
import com.sun.jersey.api.client.ClientHandlerException;
import com.sun.jersey.api.client.ClientResponse;
import com.sun.jersey.api.client.UniformInterfaceException;
import com.sun.jersey.api.client.WebResource;

import org.codehaus.jettison.json.JSONArray;
import org.codehaus.jettison.json.JSONException;
Expand Down Expand Up @@ -212,7 +209,7 @@ public DAGStatus waitForCompletionWithStatusUpdates(
@Override
public void close() throws IOException {
if (httpClient != null) {
httpClient.destroy();
httpClient.close();
httpClient = null;
}
if (timelineReaderStrategy != null) {
Expand Down Expand Up @@ -426,26 +423,14 @@ protected Map<String, VertexTaskStats> parseTaskStatsForVertexes()
@VisibleForTesting
protected JSONObject getJsonRootEntity(String url) throws TezException {
try {
WebResource wr = getCachedHttpClient().resource(url);
ClientResponse response = wr.accept(MediaType.APPLICATION_JSON_TYPE)
.type(MediaType.APPLICATION_JSON_TYPE)
.get(ClientResponse.class);

final ClientResponse.Status clientResponseStatus = response.getClientResponseStatus();
if (clientResponseStatus != ClientResponse.Status.OK) {
throw new TezException("Failed to get response from YARN Timeline:" +
" errorCode:" + clientResponseStatus + ", url:" + url);
}

return response.getEntity(JSONObject.class);
} catch (ClientHandlerException e) {
Client client = getCachedHttpClient();
WebTarget target = client.target(url);
String json = target.request(MediaType.APPLICATION_JSON_TYPE)
.accept(MediaType.APPLICATION_JSON_TYPE)
.get(String.class);
return new JSONObject(json);
} catch (Exception e) {
throw new TezException("Error processing response from YARN Timeline", e);
} catch (UniformInterfaceException e) {
throw new TezException("Error accessing content from YARN Timeline - unexpected response", e);
} catch (IllegalArgumentException e) {
throw new TezException("Error accessing content from YARN Timeline - invalid url", e);
} catch (IOException e) {
throw new TezException("Error failed to get http client", e);
}
}

Expand Down
104 changes: 21 additions & 83 deletions tez-api/src/main/java/org/apache/tez/dag/api/client/TimelineReaderFactory.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,12 +24,13 @@
import java.lang.reflect.Method;
import java.net.HttpURLConnection;
import java.net.URL;
import java.net.URLEncoder;
import java.security.GeneralSecurityException;

import javax.net.ssl.HostnameVerifier;
import javax.net.ssl.HttpsURLConnection;
import javax.net.ssl.SSLSocketFactory;
import javax.ws.rs.client.Client;
import javax.ws.rs.client.ClientBuilder;

import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
Expand All @@ -42,13 +43,11 @@
import org.apache.tez.dag.api.TezException;

import com.google.common.annotations.VisibleForTesting;
import com.sun.jersey.api.client.Client;
import com.sun.jersey.api.client.config.ClientConfig;
import com.sun.jersey.api.client.config.DefaultClientConfig;
import com.sun.jersey.client.urlconnection.HttpURLConnectionFactory;
import com.sun.jersey.client.urlconnection.URLConnectionClientHandler;
import com.sun.jersey.json.impl.provider.entity.JSONRootElementProvider;

import org.glassfish.jersey.client.ClientConfig;
import org.glassfish.jersey.client.HttpUrlConnectorProvider;
import org.glassfish.jersey.client.HttpUrlConnectorProvider.ConnectionFactory;
import org.glassfish.jersey.jackson.JacksonFeature;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

Expand Down Expand Up @@ -161,10 +160,7 @@ public TimelineReaderTokenAuthenticatedStrategy(final Configuration conf,
@Override
public Client getHttpClient() throws IOException {
Authenticator authenticator;
UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
UserGroupInformation realUgi = ugi.getRealUser();
String doAsUser;
ClientConfig clientConfig = new DefaultClientConfig(JSONRootElementProvider.App.class);
ClientConfig clientConfig = new ClientConfig().register(JacksonFeature.class);
ConnectionConfigurator connectionConfigurator = getNewConnectionConf(useHttps,
connTimeout, sslFactory);

Expand All @@ -175,20 +171,18 @@ public Client getHttpClient() throws IOException {
throw new IOException("Failed to get authenticator", e);
}

if (realUgi != null) {
doAsUser = ugi.getShortUserName();
} else {
doAsUser = null;
}
ConnectionFactory factory = url -> {
try {
return new AuthenticatedURL(authenticator)
.openConnection(url, new AuthenticatedURL.Token());
} catch (Exception e) {
throw new IOException("Hadoop Authentication failed", e);
}
};

HttpURLConnectionFactory connectionFactory;
try {
connectionFactory = new TokenAuthenticatedURLConnectionFactory(connectionConfigurator, authenticator,
doAsUser);
} catch (TezException e) {
throw new IOException("Fail to create TokenAuthenticatedURLConnectionFactory", e);
}
return new Client(new URLConnectionClientHandler(connectionFactory), clientConfig);
HttpUrlConnectorProvider connectorProvider = new HttpUrlConnectorProvider().connectionFactory(factory);
clientConfig.connectorProvider(connectorProvider);
return ClientBuilder.newClient(clientConfig);
}

private static Authenticator getTokenAuthenticator() throws TezException {
Expand All @@ -203,42 +197,6 @@ private static Authenticator getTokenAuthenticator() throws TezException {
return ReflectionUtils.createClazzInstance(authenticatorClazzName);
}

private static class TokenAuthenticatedURLConnectionFactory implements HttpURLConnectionFactory {
Copy link
Copy Markdown
Contributor

@Aggarwal-Raghav Aggarwal-Raghav Apr 14, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Why TokenAuthenticatedURLConnectionFactory was removed won't it break things on Hadoop delegation token auth?

Copy link
Copy Markdown
Contributor Author

@maheshrajus maheshrajus Apr 21, 2026 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I checked and added Hadoop delegation token auth related checks in latest commit. Please check and let me know if any other scenarios we can cover. thanks !


private final Authenticator authenticator;
private final ConnectionConfigurator connConfigurator;
private final String doAsUser;
private final AuthenticatedURL.Token token;

public TokenAuthenticatedURLConnectionFactory(ConnectionConfigurator connConfigurator,
Authenticator authenticator,
String doAsUser) throws TezException {
this.connConfigurator = connConfigurator;
this.authenticator = authenticator;
this.doAsUser = doAsUser;
this.token = ReflectionUtils.createClazzInstance(
DELEGATION_TOKEN_AUTHENTICATED_URL_TOKEN_CLASS_NAME, null, null);
}

@Override
public HttpURLConnection getHttpURLConnection(URL url) throws IOException {
try {
AuthenticatedURL authenticatedURL= ReflectionUtils.createClazzInstance(
DELEGATION_TOKEN_AUTHENTICATED_URL_CLAZZ_NAME, new Class[] {
delegationTokenAuthenticatorClazz,
ConnectionConfigurator.class
}, new Object[] {
authenticator,
connConfigurator
});
return ReflectionUtils.invokeMethod(authenticatedURL,
delegationTokenAuthenticateURLOpenConnectionMethod, url, token, doAsUser);
} catch (Exception e) {
throw new IOException(e);
}
}
}

@Override
public void close() {
if (sslFactory != null) {
Expand All @@ -265,31 +223,11 @@ public TimelineReaderPseudoAuthenticatedStrategy(final Configuration conf,

@Override
public Client getHttpClient() {
ClientConfig config = new DefaultClientConfig(JSONRootElementProvider.App.class);
HttpURLConnectionFactory urlFactory = new PseudoAuthenticatedURLConnectionFactory(connectionConf);
return new Client(new URLConnectionClientHandler(urlFactory), config);
ClientConfig config = new ClientConfig().register(JacksonFeature.class);
return ClientBuilder.newClient(config);
}

@VisibleForTesting
protected static class PseudoAuthenticatedURLConnectionFactory implements HttpURLConnectionFactory {
private final ConnectionConfigurator connectionConf;

public PseudoAuthenticatedURLConnectionFactory(ConnectionConfigurator connectionConf) {
this.connectionConf = connectionConf;
}

@Override
public HttpURLConnection getHttpURLConnection(URL url) throws IOException {
String tokenString = (url.getQuery() == null ? "?" : "&") + "user.name=" +
URLEncoder.encode(UserGroupInformation.getCurrentUser().getShortUserName(), "UTF8");

HttpURLConnection httpURLConnection =
(HttpURLConnection) (new URL(url + tokenString)).openConnection();
this.connectionConf.configure(httpURLConnection);

return httpURLConnection;
}
}
// PseudoAuthenticatedURLConnectionFactory removed in Jersey 2 migration

@Override
public void close() {
Expand Down
8 changes: 4 additions & 4 deletions tez-api/src/test/java/org/apache/tez/dag/api/client/TestATSHttpClient.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -95,7 +95,7 @@ public void testGetDagStatusSimple() throws TezException, JSONException, IOExcep

final String jsonDagData =
"{ " +
" otherinfo: { " +
" otherInfo: { " +
" status: 'SUCCEEDED'," +
" diagnostics: 'SAMPLE_DIAGNOSTICS'," +
" counters: { counterGroups: [ " +
Expand All @@ -108,9 +108,9 @@ public void testGetDagStatusSimple() throws TezException, JSONException, IOExcep
"}";

final String jsonVertexData = "{entities:[ " +
"{otherinfo: {vertexName:'v1', numTasks:5,numFailedTasks:1,numSucceededTasks:2," +
"{otherInfo: {vertexName:'v1', numTasks:5,numFailedTasks:1,numSucceededTasks:2," +
"numKilledTasks:3,numCompletedTasks:3}}," +
"{otherinfo: {vertexName:'v2',numTasks:10,numFailedTasks:1,numSucceededTasks:5," +
"{otherInfo: {vertexName:'v2',numTasks:10,numFailedTasks:1,numSucceededTasks:5," +
"numKilledTasks:3,numCompletedTasks:4}}" +
"]}";

Expand Down Expand Up @@ -151,7 +151,7 @@ public void testGetVertexStatusSimple() throws JSONException, TezException, IOEx
Set<StatusGetOpts> statusOptions = new HashSet<StatusGetOpts>(1);
statusOptions.add(StatusGetOpts.GET_COUNTERS);

final String jsonData = "{entities:[ {otherinfo:{numFailedTasks:1,numSucceededTasks:2," +
final String jsonData = "{entities:[ {otherInfo:{numFailedTasks:1,numSucceededTasks:2," +
"status:'SUCCEEDED', vertexName:'vertex1name', numTasks:4, numKilledTasks: 3, " +
"numCompletedTasks: 4, diagnostics: 'diagnostics1', " +
"counters: { counterGroups: [ " +
Expand Down
Loading