diff --git a/core/src/main/java/org/apache/hop/core/extension/HopExtensionPoint.java b/core/src/main/java/org/apache/hop/core/extension/HopExtensionPoint.java index c1d8a1e51e9..c68f78f705b 100644 --- a/core/src/main/java/org/apache/hop/core/extension/HopExtensionPoint.java +++ b/core/src/main/java/org/apache/hop/core/extension/HopExtensionPoint.java @@ -106,9 +106,10 @@ public enum HopExtensionPoint { BeforeCheckTransform("Right before a transform is about to be verified."), AfterCheckTransform("After a transform has been checked for warnings/errors."), - HopServerInit("Right before the Hop webserver starts"), - HopServerStartup("Right after the Hop webserver has started and is fully functional"), - HopServerShutdown("Right before the Hop webserver will shut down"), + HopServerInit("Right before the Hop server starts"), + HopServerStartup("Right after the Hop server has started and is fully functional"), + HopServerShutdown("Right before the Hop server will shutdown"), + HopServerTerminate("Right after the Hop server shutdown"), HopServerCalculateFilename( "Right after the server configuration filename is determined, before it is used"), diff --git a/engine/src/main/java/org/apache/hop/core/IProvidesModelerMeta.java b/engine/src/main/java/org/apache/hop/core/IProvidesModelerMeta.java index 80c6b4773ec..cc48d624dda 100644 --- a/engine/src/main/java/org/apache/hop/core/IProvidesModelerMeta.java +++ b/engine/src/main/java/org/apache/hop/core/IProvidesModelerMeta.java @@ -25,6 +25,7 @@ /** * @deprecated */ +@SuppressWarnings({"removal", "DeprecatedIsStillUsed"}) @Deprecated(since = "2.9", forRemoval = true) public interface IProvidesModelerMeta extends IProvidesDatabaseConnectionInformation { IRowMeta getRowMeta(IVariables variables, ITransformData transformData); diff --git a/engine/src/main/java/org/apache/hop/core/plugins/HopServerPluginType.java b/engine/src/main/java/org/apache/hop/core/plugins/HopServerPluginType.java index 13ca1d0446c..e9d7b9dc337 100644 --- a/engine/src/main/java/org/apache/hop/core/plugins/HopServerPluginType.java +++ b/engine/src/main/java/org/apache/hop/core/plugins/HopServerPluginType.java @@ -21,7 +21,7 @@ import org.apache.hop.core.annotations.HopServerServlet; import org.apache.hop.www.IHopServerPlugin; -/** This class represents the carte plugin type. */ +/** This class represents the Hop server plugin type. */ @PluginMainClassType(IHopServerPlugin.class) @PluginAnnotationType(HopServerServlet.class) public class HopServerPluginType extends BasePluginType { diff --git a/engine/src/main/java/org/apache/hop/pipeline/Pipeline.java b/engine/src/main/java/org/apache/hop/pipeline/Pipeline.java index c4e5564a980..49f52938cdb 100644 --- a/engine/src/main/java/org/apache/hop/pipeline/Pipeline.java +++ b/engine/src/main/java/org/apache/hop/pipeline/Pipeline.java @@ -1313,6 +1313,7 @@ public void run() { * @deprecated Make attempt to fire all registered finished listeners if possible. * @throws HopException if any errors occur during notification */ + @SuppressWarnings("removal") @Override @Deprecated(since = "2.9", forRemoval = true) public void firePipelineExecutionFinishedListeners() throws HopException { @@ -1364,6 +1365,7 @@ public void pipelineCompleted() throws HopException { * @deprecated Fires the start-event listeners (if any are registered). * @throws HopException if any errors occur during notification */ + @SuppressWarnings("removal") @Override @Deprecated(since = "2.9", forRemoval = true) public void firePipelineExecutionStartedListeners() throws HopException { @@ -1655,6 +1657,7 @@ public void stopTransform(TransformMetaDataCombi combi, boolean safeStop) { /** * @deprecated */ + @SuppressWarnings("removal") @Override @Deprecated(since = "2.9", forRemoval = true) public void firePipelineExecutionStoppedListeners() { diff --git a/engine/src/main/java/org/apache/hop/pipeline/engines/remote/RemotePipelineEngine.java b/engine/src/main/java/org/apache/hop/pipeline/engines/remote/RemotePipelineEngine.java index 89576e6fb92..e4d73100c22 100644 --- a/engine/src/main/java/org/apache/hop/pipeline/engines/remote/RemotePipelineEngine.java +++ b/engine/src/main/java/org/apache/hop/pipeline/engines/remote/RemotePipelineEngine.java @@ -88,6 +88,7 @@ import org.apache.hop.www.PrepareExecutionPipelineServlet; import org.apache.hop.www.RegisterPackageServlet; import org.apache.hop.www.RegisterPipelineServlet; +import org.apache.hop.www.RemoteHopServer; import org.apache.hop.www.SniffTransformServlet; import org.apache.hop.www.StartExecutionPipelineServlet; import org.apache.hop.www.WebResult; @@ -127,7 +128,7 @@ public class RemotePipelineEngine extends Variables implements IPipelineEngine

parentPipeline; @@ -243,10 +244,12 @@ public void prepareExecution() throws HopException { serverPollInterval = Const.toLong(resolve(remotePipelineRunConfiguration.getServerPollInterval()), 2000L); - hopServer = metadataProvider.getSerializer(HopServerMeta.class).load(hopServerName); - if (hopServer == null) { + HopServerMeta hopServerMeta = + metadataProvider.getSerializer(HopServerMeta.class).load(hopServerName); + if (hopServerMeta == null) { throw new HopException("Hop server '" + hopServerName + "' could not be found"); } + hopServer = new RemoteHopServer(hopServerMeta); PipelineExecutionConfiguration pipelineExecutionConfiguration = new PipelineExecutionConfiguration(); @@ -309,7 +312,7 @@ private void sendToHopServer( executionConfiguration.getParametersMap().putAll(params); - hopServer.getLogChannel().setLogLevel(executionConfiguration.getLogLevel()); + hopServer.getLog().setLogLevel(executionConfiguration.getLogLevel()); try { if (remotePipelineRunConfiguration.isExportingResources()) { @@ -503,7 +506,7 @@ public void run() { private synchronized void getPipelineStatus() throws RuntimeException { try { HopServerPipelineStatus pipelineStatus = - hopServer.getPipelineStatus(this, subject.getName(), containerId, lastLogLineNr); + hopServer.requestPipelineStatus(this, subject.getName(), containerId, lastLogLineNr); synchronized (engineMetrics) { hasHaltedComponents = false; engineMetrics.setStartDate(pipelineStatus.getExecutionStartDate()); @@ -656,7 +659,7 @@ public void waitUntilFinished() { @Override public void stopAll() { try { - hopServer.stopPipeline(this, subject.getName(), containerId); + hopServer.requestStopPipeline(this, subject.getName(), containerId); getPipelineStatus(); } catch (Exception e) { throw new RuntimeException( @@ -672,7 +675,7 @@ public boolean hasHaltedComponents() { @Override public void pauseExecution() { try { - hopServer.pauseResumePipeline(this, subject.getName(), containerId); + hopServer.requestPauseResumePipeline(this, subject.getName(), containerId); getPipelineStatus(); } catch (Exception e) { throw new RuntimeException( @@ -1056,6 +1059,7 @@ public void removeExecutionStoppedListener( * @deprecated * @throws HopException */ + @SuppressWarnings("removal") @Override @Deprecated(since = "2.9", forRemoval = true) public void firePipelineExecutionStartedListeners() throws HopException { @@ -1071,6 +1075,7 @@ public void firePipelineExecutionStartedListeners() throws HopException { * @deprecated * @throws HopException */ + @SuppressWarnings("removal") @Override @Deprecated(since = "2.9", forRemoval = true) public void firePipelineExecutionFinishedListeners() throws HopException { @@ -1100,6 +1105,7 @@ public void fireExecutionFinishedListeners() throws HopException { * @deprecated * @throws HopException */ + @SuppressWarnings("removal") @Override @Deprecated(since = "2.9", forRemoval = true) public void firePipelineExecutionStoppedListeners() throws HopException { @@ -1332,22 +1338,6 @@ public void setLogChannel(ILogChannel log) { this.logChannel = log; } - /** - * Gets Hop server metadata - * - * @return value of Hop server - */ - public HopServerMeta getHopServer() { - return hopServer; - } - - /** - * @param hopServer The hopServer to set - */ - public void setHopServer(HopServerMeta hopServer) { - this.hopServer = hopServer; - } - /** * Gets engineMetrics * diff --git a/engine/src/main/java/org/apache/hop/server/HopServerMeta.java b/engine/src/main/java/org/apache/hop/server/HopServerMeta.java index b03331e2499..47e313baf13 100644 --- a/engine/src/main/java/org/apache/hop/server/HopServerMeta.java +++ b/engine/src/main/java/org/apache/hop/server/HopServerMeta.java @@ -114,9 +114,10 @@ public class HopServerMeta extends HopMetadataBase implements Cloneable, IXml, I private static final Class PKG = HopServerMeta.class; public static final String STRING_HOP_SERVER = "Hop Server"; private static final Random RANDOM = new Random(); - public static final String XML_TAG = "hop-server"; private static final String HTTP = "http"; private static final String HTTPS = "https"; + + public static final String XML_TAG = "hop-server"; public static final String SSL_MODE_TAG = "sslMode"; public static final int HOP_SERVER_RETRIES = getNumberOfHopServerRetries(); public static final int HOP_SERVER_RETRY_BACKOFF_INCREMENTS = getBackoffIncrements(); @@ -125,26 +126,9 @@ public class HopServerMeta extends HopMetadataBase implements Cloneable, IXml, I private static final String CONST_XML = "&xml=Y"; private static final String CONST_ID = "&id="; - private static int getNumberOfHopServerRetries() { - try { - return Integer.parseInt(Const.NVL(System.getProperty("HOP_SERVER_RETRIES"), "0")); - } catch (Exception e) { - return 0; - } - } - - public static int getBackoffIncrements() { - try { - return Integer.parseInt( - Const.NVL(System.getProperty("HOP_SERVER_RETRY_BACKOFF_INCREMENTS"), "1000")); - } catch (Exception e) { - return 1000; - } - } - private ILogChannel log; - @Setter @Getter @HopMetadataProperty private String hostname; + @HopMetadataProperty private String hostname; @HopMetadataProperty private String port; @@ -166,6 +150,10 @@ public static int getBackoffIncrements() { @HopMetadataProperty private String nonProxyHosts; + @HopMetadataProperty private String propertiesMasterName; + + @HopMetadataProperty private boolean overrideExistingProperties; + private final Date changedDate; @HopMetadataProperty private boolean sslMode; @@ -235,7 +223,6 @@ public HopServerMeta( this.proxyPort = proxyPort; this.nonProxyHosts = nonProxyHosts; this.sslMode = sslMode; - this.log = new LogChannel(this); } public HopServerMeta(Node node) { @@ -256,6 +243,9 @@ public HopServerMeta(Node node) { this.proxyHostname = XmlHandler.getTagValue(node, "proxy_hostname"); this.proxyPort = XmlHandler.getTagValue(node, "proxy_port"); this.nonProxyHosts = XmlHandler.getTagValue(node, "non_proxy_hosts"); + this.propertiesMasterName = XmlHandler.getTagValue(node, "get_properties_from_master"); + this.overrideExistingProperties = + "Y".equalsIgnoreCase(XmlHandler.getTagValue(node, "override_existing_properties")); this.log = new LogChannel(this); setSslMode("Y".equalsIgnoreCase(XmlHandler.getTagValue(node, SSL_MODE_TAG))); @@ -266,10 +256,6 @@ public HopServerMeta(Node node) { } } - public ILogChannel getLogChannel() { - return log; - } - @Override public String getXml(IVariables variables) { StringBuilder xml = new StringBuilder(); @@ -307,6 +293,7 @@ public HopServerMeta clone() { return new HopServerMeta(this); } + @SuppressWarnings("CopyConstructorMissesField") public HopServerMeta(HopServerMeta server) { this(); replaceMeta(server); @@ -330,17 +317,6 @@ public String toString() { return name; } - public String getServerAndPort(IVariables variables) { - String realHostname; - - realHostname = variables.resolve(hostname); - - if (!Utils.isEmpty(realHostname)) { - return realHostname + getPortSpecification(variables); - } - return STRING_HOP_SERVER; - } - @Override public boolean equals(Object obj) { if (!(obj instanceof HopServerMeta server)) { @@ -429,12 +405,12 @@ public String sendXml(IVariables variables, String xml, String service) throws E } } - public String sendJson(IVariables variables, String json, String service) throws Exception { + public void sendJson(IVariables variables, String json, String service) throws Exception { String encoding = Const.XML_ENCODING; HttpPost method = buildSendMethod(variables, json.getBytes(encoding), encoding, service, "application/json"); try { - return executeAuth(variables, method); + executeAuth(variables, method); } finally { // Release current connection to the connection pool once you are done method.releaseConnection(); @@ -542,10 +518,10 @@ public String sendExport(IVariables variables, String filename, String type, Str /** * Executes method with authentication. * - * @param method - * @return - * @throws IOException - * @throws ClientProtocolException + * @param method The method to execute + * @return The response + * @throws IOException In case there was a network error + * @throws ClientProtocolException In case there was a protocol problem * @throws HopException if response not ok */ private String executeAuth(IVariables variables, HttpUriRequest method) @@ -943,12 +919,11 @@ public void setDescription(String description) { * * @param hopServers the hop servers to check against. * @param oldname the old name of the hop server - * @return the new hop server name */ - public String verifyAndModifyHopServerName(List hopServers, String oldname) { + public void verifyAndModifyHopServerName(List hopServers, String oldname) { String name = getName(); if (name.equalsIgnoreCase(oldname)) { - return name; // nothing to see here: move along! + return; // nothing to see here: move along! } int nr = 2; @@ -956,7 +931,6 @@ public String verifyAndModifyHopServerName(List hopServers, Strin setName(name + " " + nr); nr++; } - return getName(); } /** @@ -969,7 +943,7 @@ public String verifyAndModifyHopServerName(List hopServers, Strin * @param lines lines number * @param type transform type * @return xml with row metadata and data - * @throws Exception + * @throws Exception In case the transform couldn't be sniffed. */ public String sniffTransform( IVariables variables, @@ -1039,34 +1013,31 @@ public void monitorRemotePipeline( boolean allFinished = false; while (!allFinished && errors == 0) { allFinished = true; - errors = 0L; // Check the remote server - if (allFinished && errors == 0) { - try { - HopServerPipelineStatus pipelineStatus = - getPipelineStatus(variables, pipelineName, serverObjectId, 0); - if (pipelineStatus.isRunning()) { - if (log.isDetailed()) { - log.logDetailed(pipelineName, "Remote pipeline is still running."); - } - allFinished = false; - } else { - if (log.isDetailed()) { - log.logDetailed(pipelineName, "Remote pipeline has finished."); - } + try { + HopServerPipelineStatus pipelineStatus = + getPipelineStatus(variables, pipelineName, serverObjectId, 0); + if (pipelineStatus.isRunning()) { + if (log.isDetailed()) { + log.logDetailed(pipelineName, "Remote pipeline is still running."); + } + allFinished = false; + } else { + if (log.isDetailed()) { + log.logDetailed(pipelineName, "Remote pipeline has finished."); } - Result result = pipelineStatus.getResult(); - errors += result.getNrErrors(); - } catch (Exception e) { - errors += 1; - log.logError( - pipelineName, - "Unable to contact remote hop server '" - + this.getName() - + "' to check pipeline status : " - + e); } + Result result = pipelineStatus.getResult(); + errors += result.getNrErrors(); + } catch (Exception e) { + errors += 1; + log.logError( + pipelineName, + "Unable to contact remote hop server '" + + this.getName() + + "' to check pipeline status : " + + e); } // @@ -1120,34 +1091,31 @@ public void monitorRemoteWorkflow( boolean allFinished = false; while (!allFinished && errors == 0) { allFinished = true; - errors = 0L; // Check the remote server - if (allFinished && errors == 0) { - try { - HopServerWorkflowStatus jobStatus = - getWorkflowStatus(variables, workflowName, serverObjectId, 0); - if (jobStatus.isRunning()) { - if (log.isDetailed()) { - log.logDetailed(workflowName, "Remote workflow is still running."); - } - allFinished = false; - } else { - if (log.isDetailed()) { - log.logDetailed(workflowName, "Remote workflow has finished."); - } + try { + HopServerWorkflowStatus jobStatus = + getWorkflowStatus(variables, workflowName, serverObjectId, 0); + if (jobStatus.isRunning()) { + if (log.isDetailed()) { + log.logDetailed(workflowName, "Remote workflow is still running."); + } + allFinished = false; + } else { + if (log.isDetailed()) { + log.logDetailed(workflowName, "Remote workflow has finished."); } - Result result = jobStatus.getResult(); - errors += result.getNrErrors(); - } catch (Exception e) { - errors += 1; - log.logError( - workflowName, - "Unable to contact remote hop server '" - + this.getName() - + "' to check workflow status : " - + e); } + Result result = jobStatus.getResult(); + errors += result.getNrErrors(); + } catch (Exception e) { + errors += 1; + log.logError( + workflowName, + "Unable to contact remote hop server '" + + this.getName() + + "' to check workflow status : " + + e); } // @@ -1170,4 +1138,21 @@ public void monitorRemoteWorkflow( log.logBasic(workflowName, "The remote workflow has finished."); } + + private static int getNumberOfHopServerRetries() { + try { + return Integer.parseInt(Const.NVL(System.getProperty("HOP_SERVER_RETRIES"), "0")); + } catch (Exception e) { + return 0; + } + } + + public static int getBackoffIncrements() { + try { + return Integer.parseInt( + Const.NVL(System.getProperty("HOP_SERVER_RETRY_BACKOFF_INCREMENTS"), "1000")); + } catch (Exception e) { + return 1000; + } + } } diff --git a/engine/src/main/java/org/apache/hop/workflow/Workflow.java b/engine/src/main/java/org/apache/hop/workflow/Workflow.java index ce70bc20b7b..cf49948b3ba 100644 --- a/engine/src/main/java/org/apache/hop/workflow/Workflow.java +++ b/engine/src/main/java/org/apache/hop/workflow/Workflow.java @@ -536,6 +536,7 @@ public Result executeFromStart(int nr, Result result) throws HopException { * @deprecated * @param listener workflow started listener */ + @SuppressWarnings("removal") @Override @Deprecated(since = "2.9", forRemoval = true) public void addWorkflowStartedListener( @@ -563,6 +564,7 @@ public void removeExecutionStartedListener( * @deprecated * @throws HopException */ + @SuppressWarnings("removal") @Override @Deprecated(since = "2.9", forRemoval = true) public void fireWorkflowStartedListeners() throws HopException { @@ -583,6 +585,7 @@ public void fireExecutionStartedListeners() throws HopException { * @deprecated * @param listener */ + @SuppressWarnings("removal") @Override @Deprecated(since = "2.9", forRemoval = true) public void addWorkflowFinishedListener( @@ -610,6 +613,7 @@ public void removeExecutionFinishedListener( * @deprecated * @throws HopException */ + @SuppressWarnings("removal") @Override @Deprecated(since = "2.9", forRemoval = true) public void fireWorkflowFinishListeners() throws HopException { @@ -1496,6 +1500,7 @@ public void setLogLevel(LogLevel logLevel) { * * @return the interactive */ + @SuppressWarnings("removal") @Override public boolean isInteractive() { return true; @@ -1506,6 +1511,7 @@ public boolean isInteractive() { * * @param interactive the interactive to set */ + @SuppressWarnings("removal") @Override public void setInteractive(boolean interactive) {} @@ -1724,6 +1730,7 @@ public void setExecutionEndDate(Date executionEndDate) { * @deprecated Gets workflowFinishedListeners * @return value of workflowFinishedListeners */ + @SuppressWarnings("removal") @Override @Deprecated(since = "2.9", forRemoval = true) public List>> @@ -1745,6 +1752,7 @@ public void setWorkflowFinishedListeners( * @deprecated Gets workflowStartedListeners * @return value of workflowStartedListeners */ + @SuppressWarnings("removal") @Override @Deprecated(since = "2.9", forRemoval = true) public List>> diff --git a/engine/src/main/java/org/apache/hop/workflow/engines/remote/RemoteWorkflowEngine.java b/engine/src/main/java/org/apache/hop/workflow/engines/remote/RemoteWorkflowEngine.java index 6b7955947ce..991bf9f120b 100644 --- a/engine/src/main/java/org/apache/hop/workflow/engines/remote/RemoteWorkflowEngine.java +++ b/engine/src/main/java/org/apache/hop/workflow/engines/remote/RemoteWorkflowEngine.java @@ -25,7 +25,10 @@ import java.util.LinkedList; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Set; +import lombok.Getter; +import lombok.Setter; import org.apache.commons.lang.StringUtils; import org.apache.commons.vfs2.FileObject; import org.apache.hop.core.Const; @@ -76,6 +79,7 @@ import org.apache.hop.www.HopServerWorkflowStatus; import org.apache.hop.www.RegisterPackageServlet; import org.apache.hop.www.RegisterWorkflowServlet; +import org.apache.hop.www.RemoteHopServer; import org.apache.hop.www.WebResult; @WorkflowEnginePlugin( @@ -93,81 +97,96 @@ public class RemoteWorkflowEngine extends Variables implements IWorkflowEngine>> + @Setter @Getter protected boolean gatheringMetrics; + @Setter @Getter protected boolean forcingSeparateLogging; + + @Getter @Setter protected Date executionStartDate; + + @Getter @Setter protected Date executionEndDate; + + protected final List>> executionFinishedListeners; - protected List>> + protected final List>> executionStartedListeners; - protected List>> + protected final List>> executionStoppedListeners; + /** + * -- SETTER -- + * + * @param actionListeners The actionListeners to set + */ + @Getter + @Setter + @SuppressWarnings("rawtypes") protected List actionListeners; - protected List delegationListeners; + @Setter @Getter protected List delegationListeners; - protected Set activeActions; + @Getter @Setter protected Set activeActions; - protected Map extensionDataMap; + @Getter @Setter protected Map extensionDataMap; /** * The rows that were passed onto this workflow by a previous pipeline. These rows are passed onto * the first workflow entry in this workflow (on the result object) */ - private List sourceRows; + @Setter @Getter private List sourceRows; - /** Parameters of the workflow. */ - private INamedParameters namedParams = new NamedParameters(); + @Setter @Getter private INamedParameters namedParams = new NamedParameters(); - private ActionMeta startActionMeta; + @Setter @Getter private ActionMeta startActionMeta; /** * The workflow that's launching this (sub-) workflow. This gives us access to the whole chain, * including the parent variables, etc. */ - protected IWorkflowEngine parentWorkflow; + @Setter @Getter protected IWorkflowEngine parentWorkflow; /** The parent pipeline */ - protected IPipelineEngine parentPipeline; + @Getter @Setter protected IPipelineEngine parentPipeline; /** The parent logging interface to reference */ - private ILoggingObject parentLoggingObject; + @Setter @Getter private ILoggingObject parentLoggingObject; - /** - * Keep a list of the actions that were executed. - * org.apache.hop.core.logging.CentralLogStore.getInstance() - */ - private WorkflowTracker workflowTracker; + /** Keep a list of the actions that were executed. */ + @Getter @Setter private WorkflowTracker workflowTracker; /** A flat list of results in THIS workflow, in the order of execution of actions */ - private final LinkedList actionResults = new LinkedList<>(); + @Getter private final LinkedList actionResults = new LinkedList<>(); public RemoteWorkflowEngine() { executionStartedListeners = Collections.synchronizedList(new ArrayList<>()); @@ -178,7 +197,7 @@ public RemoteWorkflowEngine() { extensionDataMap = new HashMap<>(); logChannel = LogChannel.GENERAL; logLevel = LogLevel.BASIC; - workflowTracker = new WorkflowTracker(null); + workflowTracker = new WorkflowTracker<>(null); } @Override @@ -212,13 +231,9 @@ public Result startExecution() { loggingObject = new LoggingObject(this); logLevel = logChannel.getLogLevel(); - workflowTracker = new WorkflowTracker(workflowMeta); + workflowTracker = new WorkflowTracker<>(workflowMeta); - if (previousResult == null) { - result = new Result(); - } else { - result = previousResult; - } + result = Objects.requireNonNullElseGet(previousResult, Result::new); IWorkflowEngineRunConfiguration engineRunConfiguration = workflowRunConfiguration.getEngineRunConfiguration(); @@ -255,23 +270,14 @@ public Result startExecution() { + workflowRunConfiguration.getName() + "'"); - hopServer = metadataProvider.getSerializer(HopServerMeta.class).load(hopServerName); - if (hopServer == null) { + HopServerMeta hopServerMeta = + metadataProvider.getSerializer(HopServerMeta.class).load(hopServerName); + if (hopServerMeta == null) { throw new HopException("Hop server '" + hopServerName + "' could not be found"); } WorkflowExecutionConfiguration workflowExecutionConfiguration = - new WorkflowExecutionConfiguration(); - workflowExecutionConfiguration.setRunConfiguration(remoteRunConfigurationName); - if (logLevel != null) { - workflowExecutionConfiguration.setLogLevel(logLevel); - } - if (previousResult != null) { - // This contains result rows, files, ... - // - workflowExecutionConfiguration.setPreviousResult(previousResult); - } - workflowExecutionConfiguration.setGatheringMetrics(gatheringMetrics); + getWorkflowExecutionConfiguration(remoteRunConfigurationName); sendToHopServer(this, workflowMeta, workflowExecutionConfiguration, metadataProvider); fireExecutionStartedListeners(); @@ -296,6 +302,23 @@ public Result startExecution() { return result; } + private WorkflowExecutionConfiguration getWorkflowExecutionConfiguration( + String remoteRunConfigurationName) { + WorkflowExecutionConfiguration workflowExecutionConfiguration = + new WorkflowExecutionConfiguration(); + workflowExecutionConfiguration.setRunConfiguration(remoteRunConfigurationName); + if (logLevel != null) { + workflowExecutionConfiguration.setLogLevel(logLevel); + } + if (previousResult != null) { + // This contains result rows, files, ... + // + workflowExecutionConfiguration.setPreviousResult(previousResult); + } + workflowExecutionConfiguration.setGatheringMetrics(gatheringMetrics); + return workflowExecutionConfiguration; + } + public void monitorRemoteWorkflowUntilFinished() { try { // Start with a little bit of a wait @@ -324,16 +347,15 @@ public synchronized void getWorkflowStatus() throws HopException { } try { workflowStatus = - hopServer.getWorkflowStatus(this, workflowMeta.getName(), containerId, lastLogLineNr); + hopServer.requestWorkflowStatus(this, workflowMeta.getName(), containerId, lastLogLineNr); lastLogLineNr = workflowStatus.getLastLoggingLineNr(); if (StringUtils.isNotEmpty(workflowStatus.getLoggingString())) { - // TODO implement detailed logging and add option to log at all logChannel.logBasic(workflowStatus.getLoggingString()); } finished = workflowStatus.isFinished(); stopped = workflowStatus.isStopped(); running = workflowStatus.isRunning(); - active = running; // TODO: differentiate + active = running; statusDescription = workflowStatus.getStatusDescription(); result = workflowStatus.getResult(); @@ -361,7 +383,7 @@ public synchronized void getWorkflowStatus() throws HopException { @Override public void stopExecution() { try { - hopServer.stopWorkflow(this, workflowMeta.getName(), containerId); + hopServer.requestStopWorkflow(this, workflowMeta.getName(), containerId); getWorkflowStatus(); fireExecutionStoppedListeners(); @@ -399,7 +421,7 @@ public void sendToHopServer( } // Align logging levels between execution configuration and remote server - hopServer.getLogChannel().setLogLevel(executionConfiguration.getLogLevel()); + hopServer.getLog().setLogLevel(executionConfiguration.getLogLevel()); try { // Add current variables to the configuration @@ -420,36 +442,37 @@ public void sendToHopServer( if (remoteWorkflowRunConfiguration.isExportingResources()) { // First export the workflow... // - FileObject tempFile = - HopVfs.createTempFile("workflowExport", ".zip", System.getProperty("java.io.tmpdir")); - - TopLevelResource topLevelResource = - ResourceUtil.serializeResourceExportInterface( - tempFile.getName().toString(), - workflowMeta, - this, - metadataProvider, - executionConfiguration, - CONFIGURATION_IN_EXPORT_FILENAME, - remoteWorkflowRunConfiguration.getNamedResourcesSourceFolder(), - remoteWorkflowRunConfiguration.getNamedResourcesTargetFolder(), - executionConfiguration.getVariablesMap()); - - // Send the zip file over to the hop server... - String result = - hopServer.sendExport( - this, - topLevelResource.getArchiveName(), - RegisterPackageServlet.TYPE_WORKFLOW, - topLevelResource.getBaseResourceName()); - WebResult webResult = WebResult.fromXmlString(result); - if (!webResult.getResult().equalsIgnoreCase(WebResult.STRING_OK)) { - throw new HopException( - "There was an error passing the exported workflow to the remote server: " - + Const.CR - + webResult.getMessage()); + try (FileObject tempFile = + HopVfs.createTempFile("workflowExport", ".zip", System.getProperty("java.io.tmpdir"))) { + + TopLevelResource topLevelResource = + ResourceUtil.serializeResourceExportInterface( + tempFile.getName().toString(), + workflowMeta, + this, + metadataProvider, + executionConfiguration, + CONFIGURATION_IN_EXPORT_FILENAME, + remoteWorkflowRunConfiguration.getNamedResourcesSourceFolder(), + remoteWorkflowRunConfiguration.getNamedResourcesTargetFolder(), + executionConfiguration.getVariablesMap()); + + // Send the zip file over to the hop server... + String result = + hopServer.sendExport( + this, + topLevelResource.getArchiveName(), + RegisterPackageServlet.TYPE_WORKFLOW, + topLevelResource.getBaseResourceName()); + WebResult webResult = WebResult.fromXmlString(result); + if (!webResult.getResult().equalsIgnoreCase(WebResult.STRING_OK)) { + throw new HopException( + "There was an error passing the exported workflow to the remote server: " + + Const.CR + + webResult.getMessage()); + } + containerId = webResult.getId(); } - containerId = webResult.getId(); } else { String xml = new WorkflowConfiguration(workflowMeta, executionConfiguration, metadataProvider) @@ -469,7 +492,8 @@ public void sendToHopServer( // Start the workflow // - WebResult webResult = hopServer.startWorkflow(this, workflowMeta.getName(), containerId); + WebResult webResult = + hopServer.requestStartWorkflow(this, workflowMeta.getName(), containerId); if (!webResult.getResult().equalsIgnoreCase(WebResult.STRING_OK)) { throw new HopException( "There was an error starting the workflow on the remote server: " @@ -483,62 +507,6 @@ public void sendToHopServer( } } - /** - * @deprecated - * @param finishedListener - */ - @Override - @Deprecated(since = "2.9", forRemoval = true) - public void addWorkflowFinishedListener( - IExecutionFinishedListener> finishedListener) { - synchronized (executionFinishedListeners) { - executionFinishedListeners.add(finishedListener); - } - } - - /** - * @deprecated - * @throws HopException - */ - @Override - @Deprecated(since = "2.9", forRemoval = true) - public void fireWorkflowFinishListeners() throws HopException { - synchronized (executionFinishedListeners) { - for (IExecutionFinishedListener> listener : - executionFinishedListeners) { - listener.finished(this); - } - } - } - - /** - * @deprecated - * @param finishedListener - */ - @Override - @Deprecated(since = "2.9", forRemoval = true) - public void addWorkflowStartedListener( - IExecutionStartedListener> finishedListener) { - synchronized (executionStartedListeners) { - executionStartedListeners.add(finishedListener); - } - } - - /** - * @deprecated - * @throws HopException - */ - @Override - @Deprecated(since = "2.9", forRemoval = true) - public void fireWorkflowStartedListeners() throws HopException { - synchronized (executionStartedListeners) { - for (IExecutionStartedListener> listener : - executionStartedListeners) { - listener.started(this); - } - } - } - @Override public void addExecutionStartedListener( IExecutionStartedListener> listener) { @@ -618,11 +586,11 @@ public void fireExecutionStoppedListeners() { } @Override - public void addActionListener(IActionListener actionListener) { + public void addActionListener(IActionListener actionListener) { actionListeners.add(actionListener); } - public void removeActionListener(IActionListener actionListener) { + public void removeActionListener(IActionListener actionListener) { actionListeners.remove(actionListener); } @@ -647,9 +615,9 @@ public String getLogChannelId() { } /** - * Gets LoggingObjectType.JOB, which is always the value for Workflow. + * Return value 'LoggingObjectType.WORKFLOW' which is always the value for Workflow. * - * @return LoggingObjectType LoggingObjectType.JOB + * @return LoggingObjectType Always returns the workflow type. */ @Override public LoggingObjectType getObjectType() { @@ -796,49 +764,6 @@ public void copyParametersFromDefinitions(INamedParameterDefinitions definitions namedParams.copyParametersFromDefinitions(definitions); } - /** - * Gets workflowMeta - * - * @return value of workflowMeta - */ - @Override - public WorkflowMeta getWorkflowMeta() { - return workflowMeta; - } - - /** - * @param workflowMeta The workflowMeta to set - */ - @Override - public void setWorkflowMeta(WorkflowMeta workflowMeta) { - this.workflowMeta = workflowMeta; - } - - /** - * Gets pluginId - * - * @return value of pluginId - */ - public String getPluginId() { - return pluginId; - } - - /** - * @param pluginId The pluginId to set - */ - public void setPluginId(String pluginId) { - this.pluginId = pluginId; - } - - /** - * Gets workflowRunConfiguration - * - * @return value of workflowRunConfiguration - */ - public WorkflowRunConfiguration getWorkflowRunConfiguration() { - return workflowRunConfiguration; - } - /** * @param workflowRunConfiguration The workflowRunConfiguration to set */ @@ -847,325 +772,11 @@ public void setWorkflowRunConfiguration(WorkflowRunConfiguration workflowRunConf this.workflowRunConfiguration = workflowRunConfiguration; } - /** - * Gets previousResult - * - * @return value of previousResult - */ - public Result getPreviousResult() { - return previousResult; - } - - /** - * @param previousResult The previousResult to set - */ - public void setPreviousResult(Result previousResult) { - this.previousResult = previousResult; - } - - /** - * Gets result - * - * @return value of result - */ - @Override - public Result getResult() { - return result; - } - - /** - * @param result The result to set - */ - @Override - public void setResult(Result result) { - this.result = result; - } - - /** - * Gets metadataProvider - * - * @return value of metadataProvider - */ - @Override - public IHopMetadataProvider getMetadataProvider() { - return metadataProvider; - } - - /** - * @param metadataProvider The metadataProvider to set - */ - @Override - public void setMetadataProvider(IHopMetadataProvider metadataProvider) { - this.metadataProvider = metadataProvider; - } - - /** - * Gets logChannel - * - * @return value of logChannel - */ - @Override - public ILogChannel getLogChannel() { - return logChannel; - } - - /** - * @param logChannel The logChannel to set - */ - public void setLogChannel(ILogChannel logChannel) { - this.logChannel = logChannel; - } - - /** - * Gets loggingObject - * - * @return value of loggingObject - */ - public LoggingObject getLoggingObject() { - return loggingObject; - } - - /** - * @param loggingObject The loggingObject to set - */ - public void setLoggingObject(LoggingObject loggingObject) { - this.loggingObject = loggingObject; - } - - /** - * Gets logLevel - * - * @return value of logLevel - */ - @Override - public LogLevel getLogLevel() { - return logLevel; - } - - /** - * @param logLevel The logLevel to set - */ - @Override - public void setLogLevel(LogLevel logLevel) { - this.logLevel = logLevel; - } - - /** - * Gets hopServer - * - * @return value of hopServer - */ - public HopServerMeta getHopServer() { - return hopServer; - } - - /** - * @param hopServer The hopServer to set - */ - public void setHopServer(HopServerMeta hopServer) { - this.hopServer = hopServer; - } - - /** - * Gets serverObjectId - * - * @return value of serverObjectId - */ - @Override - public String getContainerId() { - return containerId; - } - - /** - * @param containerId The serverObjectId to set - */ - @Override - public void setContainerId(String containerId) { - this.containerId = containerId; - } - - /** - * Gets lastLogLineNr - * - * @return value of lastLogLineNr - */ - public int getLastLogLineNr() { - return lastLogLineNr; - } - - /** - * @param lastLogLineNr The lastLogLineNr to set - */ - public void setLastLogLineNr(int lastLogLineNr) { - this.lastLogLineNr = lastLogLineNr; - } - - /** - * Gets stopped - * - * @return value of stopped - */ - @Override - public boolean isStopped() { - return stopped; - } - - /** - * @param stopped The stopped to set - */ - @Override - public void setStopped(boolean stopped) { - this.stopped = stopped; - } - - /** - * @param workflowStatus The workflowStatus to set - */ - public void setWorkflowStatus(HopServerWorkflowStatus workflowStatus) { - this.workflowStatus = workflowStatus; - } - - /** - * Gets interactive - * - * @return value of interactive - */ - @Override - public boolean isInteractive() { - return interactive; - } - - /** - * @param interactive The interactive to set - */ - @Override - public void setInteractive(boolean interactive) { - this.interactive = interactive; - } - - /** - * Gets finished - * - * @return value of finished - */ - @Override - public boolean isFinished() { - return finished; - } - - /** - * @param finished The finished to set - */ - @Override - public void setFinished(boolean finished) { - this.finished = finished; - } - - /** - * Gets initialized - * - * @return value of initialized - */ - @Override - public boolean isInitialized() { - return initialized; - } - - /** - * @param initialized The initialized to set - */ - public void setInitialized(boolean initialized) { - this.initialized = initialized; - } - - /** - * Gets running - * - * @return value of running - */ - public boolean isRunning() { - return running; - } - - /** - * @param running The running to set - */ - public void setRunning(boolean running) { - this.running = running; - } - - /** - * Gets statusDescription - * - * @return value of statusDescription - */ - @Override - public String getStatusDescription() { - return statusDescription; - } - - /** - * @param statusDescription The statusDescription to set - */ - public void setStatusDescription(String statusDescription) { - this.statusDescription = statusDescription; - } - - /** - * Gets active - * - * @return value of active - */ - @Override - public boolean isActive() { - return active; - } - - /** - * @param active The active to set - */ - public void setActive(boolean active) { - this.active = active; - } - - /** - * Gets executionStartDate - * - * @return value of executionStartDate - */ - @Override - public Date getExecutionStartDate() { - return executionStartDate; - } - - /** - * @param executionStartDate The executionStartDate to set - */ - public void setExecutionStartDate(Date executionStartDate) { - this.executionStartDate = executionStartDate; - } - - /** - * Gets executionEndDate - * - * @return value of executionEndDate - */ - @Override - public Date getExecutionEndDate() { - return executionEndDate; - } - - /** - * @param executionEndDate The executionEndDate to set - */ - public void setExecutionEndDate(Date executionEndDate) { - this.executionEndDate = executionEndDate; - } - /** * @deprecated Gets workflowFinishedListeners * @return value of workflowFinishedListeners */ + @SuppressWarnings("removal") @Override @Deprecated(since = "2.9", forRemoval = true) public List>> @@ -1180,13 +791,15 @@ public void setExecutionEndDate(Date executionEndDate) { @Deprecated(since = "2.9", forRemoval = true) public void setWorkflowFinishedListeners( List>> workflowFinishedListeners) { - this.executionFinishedListeners = workflowFinishedListeners; + this.executionFinishedListeners.clear(); + this.executionFinishedListeners.addAll(workflowFinishedListeners); } /** * @deprecated Gets workflowStartedListeners * @return value of workflowStartedListeners */ + @SuppressWarnings("removal") @Override @Deprecated(since = "2.9", forRemoval = true) public List>> @@ -1201,238 +814,87 @@ public void setWorkflowFinishedListeners( @Deprecated(since = "2.9", forRemoval = true) public void setWorkflowStartedListeners( List>> workflowStartedListeners) { - this.executionStartedListeners = workflowStartedListeners; - } - - /** - * Gets actionListeners - * - * @return value of actionListeners - */ - @Override - public List getActionListeners() { - return actionListeners; - } - - /** - * @param actionListeners The actionListeners to set - */ - public void setActionListeners(List actionListeners) { - this.actionListeners = actionListeners; - } - - /** - * Gets delegationListeners - * - * @return value of delegationListeners - */ - public List getDelegationListeners() { - return delegationListeners; - } - - /** - * @param delegationListeners The delegationListeners to set - */ - public void setDelegationListeners(List delegationListeners) { - this.delegationListeners = delegationListeners; - } - - /** - * Gets activeActionPipeline - * - * @return value of activeActionPipeline - */ - @Override - public Set getActiveActions() { - return activeActions; - } - - /** - * @param activeActions The activeActions to set - */ - public void setActiveActions(Set activeActions) { - this.activeActions = activeActions; - } - - /** - * Gets namedParams - * - * @return value of namedParams - */ - public INamedParameters getNamedParams() { - return namedParams; + this.executionStartedListeners.clear(); + this.executionStartedListeners.addAll(workflowStartedListeners); } /** - * @param namedParams The namedParams to set - */ - public void setNamedParams(INamedParameters namedParams) { - this.namedParams = namedParams; - } - - /** - * Gets extensionDataMap - * - * @return value of extensionDataMap - */ - @Override - public Map getExtensionDataMap() { - return extensionDataMap; - } - - /** - * @param extensionDataMap The extensionDataMap to set - */ - public void setExtensionDataMap(Map extensionDataMap) { - this.extensionDataMap = extensionDataMap; - } - - /** - * Gets sourceRows - * - * @return value of sourceRows - */ - public List getSourceRows() { - return sourceRows; - } - - /** - * @param sourceRows The sourceRows to set - */ - @Override - public void setSourceRows(List sourceRows) { - this.sourceRows = sourceRows; - } - - /** - * Gets start action meta - * - * @return value of start action meta - */ - public ActionMeta getStartActionMeta() { - return startActionMeta; - } - - /** - * @param actionMeta The start action to set - */ - @Override - public void setStartActionMeta(ActionMeta actionMeta) { - this.startActionMeta = actionMeta; - } - - /** - * Gets workflowTracker - * - * @return value of workflowTracker - */ - @Override - public WorkflowTracker getWorkflowTracker() { - return workflowTracker; - } - - /** - * @param workflowTracker The workflowTracker to set - */ - public void setWorkflowTracker(WorkflowTracker workflowTracker) { - this.workflowTracker = workflowTracker; - } - - /** - * Gets parentWorkflow - * - * @return value of parentWorkflow - */ - @Override - public IWorkflowEngine getParentWorkflow() { - return parentWorkflow; - } - - /** - * @param parentWorkflow The parentWorkflow to set - */ - @Override - public void setParentWorkflow(IWorkflowEngine parentWorkflow) { - this.parentWorkflow = parentWorkflow; - } - - /** - * Gets parentPipeline - * - * @return value of parentPipeline - */ - @Override - public IPipelineEngine getParentPipeline() { - return parentPipeline; - } - - /** - * @param parentPipeline The parentPipeline to set + * @deprecated + * @param finishedListener The listener to add */ + @SuppressWarnings("removal") @Override - public void setParentPipeline(IPipelineEngine parentPipeline) { - this.parentPipeline = parentPipeline; - } - - /** - * Gets parentLoggingObject - * - * @return value of parentLoggingObject - */ - public ILoggingObject getParentLoggingObject() { - return parentLoggingObject; - } - - /** - * @param parentLoggingObject The parentLoggingObject to set - */ - public void setParentLoggingObject(ILoggingObject parentLoggingObject) { - this.parentLoggingObject = parentLoggingObject; + @Deprecated(since = "2.9", forRemoval = true) + public void addWorkflowFinishedListener( + IExecutionFinishedListener> finishedListener) { + synchronized (executionFinishedListeners) { + executionFinishedListeners.add(finishedListener); + } } /** - * Gets actionResults - * - * @return value of actionResults + * @deprecated + * @throws HopException in case an exception happens during execution of a listener */ + @SuppressWarnings("removal") @Override - public LinkedList getActionResults() { - return actionResults; + @Deprecated(since = "2.9", forRemoval = true) + public void fireWorkflowFinishListeners() throws HopException { + synchronized (executionFinishedListeners) { + for (IExecutionFinishedListener> listener : + executionFinishedListeners) { + listener.finished(this); + } + } } /** - * Gets forcingSeparateLogging - * - * @return value of forcingSeparateLogging + * @deprecated + * @param finishedListener The listener to add */ + @SuppressWarnings("removal") @Override - public boolean isForcingSeparateLogging() { - return forcingSeparateLogging; + @Deprecated(since = "2.9", forRemoval = true) + public void addWorkflowStartedListener( + IExecutionStartedListener> finishedListener) { + synchronized (executionStartedListeners) { + executionStartedListeners.add(finishedListener); + } } /** - * @param forcingSeparateLogging The forcingSeparateLogging to set + * @deprecated + * @throws HopException in case an exception happens during execution of a listener */ + @SuppressWarnings("removal") @Override - public void setForcingSeparateLogging(boolean forcingSeparateLogging) { - this.forcingSeparateLogging = forcingSeparateLogging; + @Deprecated(since = "2.9", forRemoval = true) + public void fireWorkflowStartedListeners() throws HopException { + synchronized (executionStartedListeners) { + for (IExecutionStartedListener> listener : + executionStartedListeners) { + listener.started(this); + } + } } /** - * Gets gatheringMetrics + * Gets interactive * - * @return value of gatheringMetrics + * @return value of interactive */ + @SuppressWarnings("removal") @Override - public boolean isGatheringMetrics() { - return gatheringMetrics; + public boolean isInteractive() { + return interactive; } /** - * @param gatheringMetrics The gatheringMetrics to set + * @param interactive The interactive to set */ + @SuppressWarnings("removal") @Override - public void setGatheringMetrics(boolean gatheringMetrics) { - this.gatheringMetrics = gatheringMetrics; + public void setInteractive(boolean interactive) { + this.interactive = interactive; } } diff --git a/engine/src/main/java/org/apache/hop/www/DefaultWebServerShutdownHandler.java b/engine/src/main/java/org/apache/hop/www/DefaultWebServerShutdownHandler.java index 8181eadd31c..ad20d8c535c 100644 --- a/engine/src/main/java/org/apache/hop/www/DefaultWebServerShutdownHandler.java +++ b/engine/src/main/java/org/apache/hop/www/DefaultWebServerShutdownHandler.java @@ -6,13 +6,14 @@ * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. + * */ package org.apache.hop.www; diff --git a/engine/src/main/java/org/apache/hop/www/HopServer.java b/engine/src/main/java/org/apache/hop/www/HopServer.java index 541d45a542c..9154f9b5380 100644 --- a/engine/src/main/java/org/apache/hop/www/HopServer.java +++ b/engine/src/main/java/org/apache/hop/www/HopServer.java @@ -40,6 +40,7 @@ import org.apache.hop.core.HopVersionProvider; import org.apache.hop.core.Result; import org.apache.hop.core.config.plugin.ConfigPlugin; +import org.apache.hop.core.config.plugin.ConfigPluginType; import org.apache.hop.core.config.plugin.IConfigOptions; import org.apache.hop.core.encryption.Encr; import org.apache.hop.core.exception.HopException; @@ -48,7 +49,9 @@ import org.apache.hop.core.logging.ILogChannel; import org.apache.hop.core.logging.LogChannel; import org.apache.hop.core.logging.LogLevel; +import org.apache.hop.core.plugins.IPlugin; import org.apache.hop.core.plugins.JarCache; +import org.apache.hop.core.plugins.PluginRegistry; import org.apache.hop.core.util.Utils; import org.apache.hop.core.variables.IVariables; import org.apache.hop.core.variables.Variables; @@ -72,7 +75,6 @@ import org.w3c.dom.Node; import picocli.CommandLine; import picocli.CommandLine.Command; -import picocli.CommandLine.Parameters; @SuppressWarnings("java:S106") @Getter @@ -80,7 +82,10 @@ @Command( versionProvider = HopVersionProvider.class, mixinStandardHelpOptions = true, - description = "Run a Hop server") + name = "hop-server", + description = "Apache Hop server", + abbreviateSynopsis = true, + usageHelpAutoWidth = true) @HopCommand(id = "server", description = "Run a Hop server") public class HopServer implements Runnable, IHasHopMetadataProvider, IHopCommand { private static final Class PKG = HopServer.class; @@ -88,22 +93,55 @@ public class HopServer implements Runnable, IHasHopMetadataProvider, IHopCommand private static final String CONST_SPACE = " "; private static final String CONST_USAGE_EXAMPLE = "HopServer.Usage.Example"; - @Parameters(description = "One XML configuration file or a hostname and port", arity = "0..3") + @CommandLine.Parameters(description = "The XML configuration file", arity = "0..1") private List parameters; - @picocli.CommandLine.Option( + @CommandLine.Option( + names = {"-h", "--help"}, + usageHelp = true, + description = "Displays this help message and quits.") + private boolean helpRequested; + + @CommandLine.Option( + names = {"-v", "--version"}, + versionHelp = true, + description = "Displays version info and quits.") + boolean versionRequested; + + @CommandLine.Option( names = {"-p", "--password"}, description = - "The server password. Required for administrative operations only, not for starting the server.") + "The server password. Required for administrative operations only, not for starting the server.") private String password; - @picocli.CommandLine.Option( + @CommandLine.Option( names = {"-u", "--userName"}, description = - "The server user name. Required for administrative operations only, not for starting the server.") + "The server user name. Required for administrative operations only, not for starting the server.") private String username; - @picocli.CommandLine.Option( + @CommandLine.Option( + names = {"--host"}, + description = "The server host name or IP address to use.") + private String host; + + @CommandLine.Option( + names = {"--port"}, + description = "The server port to use.") + private String port; + + @CommandLine.Option( + names = {"--shutdownport"}, + description = "The server shutdown port to use.") + private String shutdownPort; + + @CommandLine.Option( + names = {"--ssl"}, + description = + "Use SSL connections to encrypt client/server communications using TLS protocols for increased security.") + private Boolean ssl; + + @CommandLine.Option( names = {"-l", "--level"}, description = "The debug level, one of NOTHING, ERROR, MINIMAL, BASIC, DETAILED, DEBUG, ROWLEVEL") @@ -111,28 +149,28 @@ public class HopServer implements Runnable, IHasHopMetadataProvider, IHopCommand @CommandLine.Option( names = {"-s", "--system-properties"}, - description = "A comma separated list of KEY=VALUE pairs", + description = "A comma separated list of KEY=VALUE pairs.", split = ",") private String[] systemProperties = null; @CommandLine.Option( names = {"-gs", "--general-status"}, - description = "List the general status of the server") + description = "List the general status of the server.") private boolean generalStatus; @CommandLine.Option( names = {"-ps", "--pipeline-status"}, - description = "List the status of the pipeline with this name (also specify the -id option)") + description = "List the status of the pipeline with this name (also specify the -id option).") private String pipelineName; @CommandLine.Option( names = {"-ws", "--workflow-status"}, - description = "List the status of the workflow with this name (also specify the -id option)") + description = "List the status of the workflow with this name (also specify the -id option).") private String workflowName; @CommandLine.Option( - names = {"-id"}, - description = "Specify the ID of the pipeline or workflow to query") + names = {"--id"}, + description = "Specify the ID of the pipeline or workflow to query.") private String id; @CommandLine.Option( @@ -145,20 +183,35 @@ public class HopServer implements Runnable, IHasHopMetadataProvider, IHopCommand description = "Does the Hop web server have authentication enabled") private Boolean enableAuth; + @CommandLine.Option( + names = {"-k", "--kill"}, + description = "Stop the running Hop server. Use the -u and -p options to authenticate.") + private boolean kill; + + private class ShutdownHook extends Thread { + public void run() { + log.logDetailed(BaseMessages.getString(PKG, "HopServer.Log.ShuttingDown")); + jvmExit = false; + shutdown(); + } + } + private WebServer webServer; private HopServerConfig config; private boolean allOK; private IVariables variables; - private picocli.CommandLine cmd; + private CommandLine cmd; private ILogChannel log; private MultiMetadataProvider metadataProvider; private Boolean joinOverride; private String realFilename; + private boolean jvmExit; public HopServer() { this.config = new HopServerConfig(); this.joinOverride = null; + this.jvmExit = true; HopServerMeta defaultServer = new HopServerMeta("local8080", "localhost", "8080", "8079", "cluster", "cluster"); @@ -183,27 +236,21 @@ public void initialize( } public void runHopServer() throws Exception { + log.logDetailed(BaseMessages.getString(PKG, "HopServer.Log.StartingServer")); allOK = true; HopServerSingleton.setHopServerConfig(config); - log = HopServerSingleton.getInstance().getLog(); final PipelineMap pipelineMap = HopServerSingleton.getInstance().getPipelineMap(); pipelineMap.setHopServerConfig(config); final WorkflowMap workflowMap = HopServerSingleton.getInstance().getWorkflowMap(); workflowMap.setHopServerConfig(config); - HopServerMeta hopServer = config.getHopServer(); + HopServerMeta hopServerMeta = config.getHopServer(); - String hostname = hopServer.getHostname(); - int port = WebServer.CONST_PORT; - int shutdownPort = WebServer.SHUTDOWN_PORT; - if (!Utils.isEmpty(hopServer.getPort())) { - port = parsePort(hopServer); - } - if (!Utils.isEmpty(hopServer.getShutdownPort())) { - shutdownPort = parseShutdownPort(hopServer); - } + String hostname = hopServerMeta.getHostname(); + int port = parsePort(hopServerMeta); + int shutdownPort = parseShutdownPort(hopServerMeta); if (allOK) { boolean shouldJoin = config.isJoining(); @@ -211,43 +258,107 @@ public void runHopServer() throws Exception { shouldJoin = joinOverride; } - this.webServer = - new WebServer( - log, - pipelineMap, - workflowMap, - hostname, - port, - shutdownPort, - shouldJoin, - config.getPasswordFile(), - hopServer.getSslConfig()); + Thread shutdownHook = new ShutdownHook(); + + try { + // Register a virtual-machine shutdown hook to stop the Hop server gracefully + Runtime.getRuntime().addShutdownHook(shutdownHook); + + webServer = + new WebServer( + log, + pipelineMap, + workflowMap, + hostname, + port, + shutdownPort, + shouldJoin, + config.getPasswordFile(), + hopServerMeta.getSslConfig()); + + // Right after the Hop server has started and is fully functional + try { + ExtensionPointHandler.callExtensionPoint( + log, variables, HopExtensionPoint.HopServerStartup.id, this); + } catch (Exception e) { + // Log error but continue regular operations to make sure HopServer continues to run + // properly + // + log.logError("Error calling extension point HopServerStartup", e); + } + + // Right after the Hop server shutdown + try { + ExtensionPointHandler.callExtensionPoint( + log, variables, HopExtensionPoint.HopServerTerminate.id, this); + } catch (Exception e) { + // Log error but continue regular operations to make sure HopServer continues to run + // properly + // + log.logError("Error calling extension point HopServerTerminate", e); + } + + log.logDetailed(BaseMessages.getString(PKG, "HopServer.Log.StoppedServer")); + } finally { + // Shutdown hooks cannot be removed once the shutdown sequence is started. + if (jvmExit) { + Runtime.getRuntime().removeShutdownHook(shutdownHook); + } + } } + } - ExtensionPointHandler.callExtensionPoint( - log, variables, HopExtensionPoint.HopServerShutdown.id, this); + public void shutdown() { + + if (webServer != null) { + // Right before the Hop server will shut down + try { + ExtensionPointHandler.callExtensionPoint( + log, variables, HopExtensionPoint.HopServerShutdown.id, this); + } catch (Exception e) { + // Log error but continue regular operations to make sure HopServer can be shutdown + // properly. + // + log.logError("Error calling extension point HopServerShutdown", e); + } + + webServer.stopServer(); + } } - private int parsePort(HopServerMeta hopServer) { + private int parsePort(HopServerMeta hopServerMeta) { + + if (Utils.isEmpty(hopServerMeta.getPort())) { + return WebServer.DEFAULT_PORT; + } + try { - return Integer.parseInt(hopServer.getPort()); + return Integer.parseInt(hopServerMeta.getPort()); } catch (Exception e) { log.logError( BaseMessages.getString( - PKG, "HopServer.Error.CanNotPartPort", hopServer.getHostname(), hopServer.getPort()), + PKG, + "HopServer.Error.CanNotPartPort", + hopServerMeta.getHostname(), + hopServerMeta.getPort()), e); allOK = false; } return -1; } - private int parseShutdownPort(HopServerMeta hopServer) { + private int parseShutdownPort(HopServerMeta hopServerMeta) { + + if (Utils.isEmpty(hopServerMeta.getShutdownPort())) { + return WebServer.DEFAULT_PORT; + } + try { - return Integer.parseInt(hopServer.getShutdownPort()); + return Integer.parseInt(hopServerMeta.getShutdownPort()); } catch (Exception e) { log.logError( BaseMessages.getString( - PKG, "HopServer.Error.CanNotPartShutdownPort", hopServer.getShutdownPort()), + PKG, "HopServer.Error.CanNotPartShutdownPort", hopServerMeta.getShutdownPort()), e); allOK = false; } @@ -256,11 +367,11 @@ private int parseShutdownPort(HopServerMeta hopServer) { @Override public void run() { + try { System.setProperty(Const.HOP_PLATFORM_RUNTIME, "SERVER"); log = new LogChannel("HopServer"); log.setLogLevel(determineLogLevel()); - log.logDetailed("Start of Hop Server"); // Allow plugins to modify the elements loaded so far, before the server is started // @@ -277,32 +388,53 @@ public void run() { } } - // If the server name was specified we make it look like 2 parameters were specified - // - if (CollectionUtils.isEmpty(parameters) && StringUtils.isNotEmpty(serverName)) { - setupByServerName(); - } - // Build the server configuration from the options... - // - // Load from an XML file that describes the complete configuration... - // + + // 1) Load from an XML file that describes the complete configuration... if (CollectionUtils.size(parameters) == 1) { setupByFileName(); + } else { + // 2) If the server name was specified + if (StringUtils.isNotEmpty(serverName)) { + setupByServerName(); + } } - if ((CollectionUtils.size(parameters) == 2 || (CollectionUtils.size(parameters) == 3)) - && StringUtils.isNotEmpty(parameters.get(0)) - && StringUtils.isNotEmpty(parameters.get(1))) { - String hostname = parameters.get(0); - String port = parameters.get(1); - - String shutdownPort = - CollectionUtils.size(parameters) == 3 - ? parameters.get(2) - : Integer.toString(WebServer.SHUTDOWN_PORT); + // Overwrite configuration with options in command line + HopServerMeta serverMeta = config.getHopServer(); + if (StringUtils.isNotEmpty(host)) { + serverMeta.setHostname(host); + } + if (StringUtils.isNotEmpty(port)) { + serverMeta.setPort(port); + } + if (StringUtils.isNotEmpty(shutdownPort)) { + serverMeta.setShutdownPort(shutdownPort); + } + if (StringUtils.isNotEmpty(username)) { + serverMeta.setUsername(username); + } + if (StringUtils.isNotEmpty(password)) { + serverMeta.setPassword(password); + } + if (ssl != null) { + serverMeta.setSslMode(ssl); + } - setupByHostNameAndPort(hostname, port, shutdownPort); + // Resolve variables properties + serverMeta.setHostname(variables.resolve(serverMeta.getHostname())); + serverMeta.setPort(variables.resolve(serverMeta.getPort())); + serverMeta.setUsername(variables.resolve(serverMeta.getUsername())); + serverMeta.setPassword(variables.resolve(serverMeta.getPassword())); + serverMeta.setShutdownPort(variables.resolve(serverMeta.getShutdownPort())); + serverMeta.setProxyHostname(variables.resolve(serverMeta.getProxyHostname())); + serverMeta.setProxyPort(variables.resolve(serverMeta.getProxyPort())); + serverMeta.setNonProxyHosts(variables.resolve(serverMeta.getNonProxyHosts())); + + // Check configuration + if (StringUtils.isEmpty(serverMeta.getUsername()) + || StringUtils.isEmpty(serverMeta.getPassword())) { + throw new HopException("Please specify the username and password."); } // Pass the variables and metadata provider @@ -322,27 +454,18 @@ public void run() { // Only query? // if (handleQueryOptions()) { - System.exit(0); + return; } // At long last, run the actual server... // runHopServer(); } catch (Exception e) { - throw new picocli.CommandLine.ExecutionException( + throw new CommandLine.ExecutionException( cmd, "There was an error during the startup of the Hop server", e); } } - private void setupByHostNameAndPort(String hostname, String port, String shutdownPort) { - HopServerMeta hopServer = - new HopServerMeta(hostname + ":" + port, hostname, port, shutdownPort, null, null); - - config = new HopServerConfig(); - config.setHopServer(hopServer); - config.setJoining(true); - } - private void setupByFileName() throws HopException { // Calculate the filename, allow plugins to intervene... // @@ -358,69 +481,84 @@ private void setupByServerName() throws HopException { IHopMetadataSerializer serializer = metadataProvider.getSerializer(HopServerMeta.class); String name = variables.resolve(serverName); - HopServerMeta hopServer = serializer.load(name); - if (hopServer == null) { + HopServerMeta meta = serializer.load(name); + if (meta == null) { throw new HopException( "Unable to find Hop Server '" + name + "' couldn't be found in the server metadata"); } - String hostname = variables.resolve(hopServer.getHostname()); - String port = variables.resolve(hopServer.getPort()); - String shutDownPort = variables.resolve(hopServer.getShutdownPort()); - parameters = List.of(Const.NVL(hostname, ""), Const.NVL(port, ""), Const.NVL(shutDownPort, "")); + + config.setHopServer(meta); } private boolean handleQueryOptions() { boolean queried = false; try { + + if (kill) { + + // Special case if shutdown the server started as service whit Apache Daemon. + // https://commons.apache.org/proper/commons-daemon/procrun.html + if (HopServerSingleton.getHopServer() != null) { + jvmExit = false; + HopServerSingleton.getHopServer().shutdown(); + } else { + // Create a custom server meta with shutdown port + HopServerMeta serverMeta = new HopServerMeta(config.getHopServer()); + serverMeta.setPort(config.getHopServer().getShutdownPort()); + + log.logBasic("Send server shutdown to " + serverMeta.getName()); + + // Shutdown a remote server + RemoteHopServer remoteServer = new RemoteHopServer(serverMeta); + remoteServer.requestShutdownServer(variables); + } + + return true; + } + // Handle username / password // if (generalStatus || StringUtils.isNotEmpty(pipelineName) || StringUtils.isNotEmpty(workflowName)) { - if (StringUtils.isEmpty(username) || StringUtils.isEmpty(password)) { - throw new HopException( - "Please specify the username and password to query the server status"); - } - config.getHopServer().setUsername(variables.resolve(username)); - config.getHopServer().setPassword(variables.resolve(password)); - } - if (generalStatus) { - queried = true; - HopServerStatus status = config.getHopServer().getStatus(variables); - // List the pipelines... - // - System.out.println("Pipelines: " + status.getPipelineStatusList().size() + CONST_FOUND); - for (HopServerPipelineStatus pipelineStatus : status.getPipelineStatusList()) { - printPipelineStatus(pipelineStatus, false); - } - System.out.println(); - // List the workflows... - // - System.out.println("Workflows: " + status.getWorkflowStatusList().size() + CONST_FOUND); - for (HopServerWorkflowStatus workflowStatus : status.getWorkflowStatusList()) { - printWorkflowStatus(workflowStatus, false); - } - } else if (StringUtils.isNotEmpty(pipelineName)) { queried = true; - if (StringUtils.isEmpty(id)) { - throw new HopException( - "Please specify the ID of the pipeline execution to see its status."); - } - HopServerPipelineStatus pipelineStatus = - config.getHopServer().getPipelineStatus(variables, pipelineName, id, 0); - printPipelineStatus(pipelineStatus, true); - } else if (StringUtils.isNotEmpty(workflowName)) { - queried = true; - if (StringUtils.isEmpty(id)) { - throw new HopException( - "Please specify the ID of the workflow execution to see its status."); + + RemoteHopServer remoteServer = new RemoteHopServer(config.getHopServer()); + + if (generalStatus) { + HopServerStatus status = remoteServer.requestServerStatus(variables); + // List the pipelines... + // + System.out.println("Pipelines: " + status.getPipelineStatusList().size() + CONST_FOUND); + for (HopServerPipelineStatus pipelineStatus : status.getPipelineStatusList()) { + printPipelineStatus(pipelineStatus, false); + } + System.out.println(); + // List the workflows... + // + System.out.println("Workflows: " + status.getWorkflowStatusList().size() + CONST_FOUND); + for (HopServerWorkflowStatus workflowStatus : status.getWorkflowStatusList()) { + printWorkflowStatus(workflowStatus, false); + } + } else if (StringUtils.isNotEmpty(pipelineName)) { + if (StringUtils.isEmpty(id)) { + throw new HopException( + "Please specify the ID of the pipeline execution to see its status."); + } + HopServerPipelineStatus pipelineStatus = + remoteServer.requestPipelineStatus(variables, pipelineName, id, 0); + printPipelineStatus(pipelineStatus, true); + } else if (StringUtils.isNotEmpty(workflowName)) { + if (StringUtils.isEmpty(id)) { + throw new HopException( + "Please specify the ID of the workflow execution to see its status."); + } + HopServerWorkflowStatus workflowStatus = + remoteServer.requestWorkflowStatus(variables, workflowName, id, 0); + printWorkflowStatus(workflowStatus, true); } - HopServerWorkflowStatus workflowStatus = - config.getHopServer().getWorkflowStatus(variables, workflowName, id, 0); - printWorkflowStatus(workflowStatus, true); } - } catch (Exception e) { log.logError("Error querying server", e); System.exit(8); @@ -525,7 +663,7 @@ public void applySystemProperties() { } } - private void buildVariableSpace() { + private void buildVariables() { // Also grabs the system properties from hop.config. // variables = Variables.getADefaultVariableSpace(); @@ -543,7 +681,8 @@ public static void main(String[] args) { try { // Create the command line options... // - picocli.CommandLine cmd = new picocli.CommandLine(hopServer); + CommandLine command = new CommandLine(hopServer); + // Apply the system properties to the JVM // hopServer.applySystemProperties(); @@ -555,7 +694,7 @@ public static void main(String[] args) { // Picks up the system settings in the variables // - hopServer.buildVariableSpace(); + hopServer.buildVariables(); // Clear the jar file cache so that we don't waste memory... // @@ -569,33 +708,47 @@ public static void main(String[] args) { // Now add server configuration plugins... // - Hop.addMixinPlugins(cmd, ConfigPlugin.CATEGORY_SERVER); - hopServer.setCmd(cmd); + List configPlugins = PluginRegistry.getInstance().getPlugins(ConfigPluginType.class); + for (IPlugin configPlugin : configPlugins) { + // Load only the plugins of the "server" category + if (ConfigPlugin.CATEGORY_SERVER.equals(configPlugin.getCategory())) { + IConfigOptions configOptions = + PluginRegistry.getInstance().loadClass(configPlugin, IConfigOptions.class); + command.addMixin(configPlugin.getIds()[0], configOptions); + } + } + hopServer.setCmd(command); + Hop.addMixinPlugins(command, ConfigPlugin.CATEGORY_SERVER); + hopServer.setCmd(command); // Add optional metadata folder (legacy) // hopServer.addMetadataFolderProvider(); - // This will calculate the option values and put them in HopRun or the plugin classes + // This will calculate the option values and put them in HopServer or the plugin classes // - picocli.CommandLine.ParseResult parseResult = cmd.parseArgs(arguments); + command.parseArgs(arguments); + + if (CollectionUtils.size(hopServer.parameters) > 1) { + throw new CommandLine.ParameterException(command, "Too many parameters"); + } - if (picocli.CommandLine.printHelpIfRequested(parseResult)) { - printExtraUsageExamples(); - System.exit(1); + // Execute the command line + if (command.isUsageHelpRequested()) { + command.usage(System.out); + } else if (command.isVersionHelpRequested()) { + command.printVersionHelp(System.out); } else { hopServer.run(); - - // If we exit now it's because the server was stopped and this is not an error - // - System.exit(0); } - } catch (picocli.CommandLine.ParameterException e) { + + HopEnvironment.shutdown(); + } catch (CommandLine.ParameterException e) { System.err.println(e.getMessage()); hopServer.cmd.usage(System.err); printExtraUsageExamples(); System.exit(9); - } catch (picocli.CommandLine.ExecutionException e) { + } catch (CommandLine.ExecutionException e) { System.err.println("Error found during execution!"); System.err.println(Const.getStackTracker(e)); System.exit(1); @@ -604,6 +757,12 @@ public static void main(String[] args) { System.err.println(Const.getStackTracker(e)); System.exit(2); } + + // If we exit now it's because the server was stopped and this is not an error + // + if (hopServer.jvmExit) { + System.exit(0); + } } private void addMetadataFolderProvider() { @@ -621,17 +780,17 @@ private void addMetadataFolderProvider() { private static void printExtraUsageExamples() { System.err.println(); System.err.println( - BaseMessages.getString(PKG, CONST_USAGE_EXAMPLE) + ": hop-server.sh 0.0.0.0 8080"); + BaseMessages.getString(PKG, CONST_USAGE_EXAMPLE) + + ": hop-server.sh --host 0.0.0.0 --port 8080"); System.err.println( BaseMessages.getString(PKG, CONST_USAGE_EXAMPLE) - + ": hop-server.sh 192.168.1.221 8081 8082"); - System.err.println(); + + ": hop-server.sh -host=192.168.1.221 -port=8081 -shutdownport=8082"); System.err.println( BaseMessages.getString(PKG, CONST_USAGE_EXAMPLE) - + ": hop-server.sh -e aura-gcp gs://apachehop/hop-server-config.xml"); + + ": hop-server.sh --environment aura-gcp gs://apachehop/hop-server-config.xml"); System.err.println( BaseMessages.getString(PKG, CONST_USAGE_EXAMPLE) - + ": hop-server.sh 127.0.0.1 8080 --kill --userName cluster --password cluster"); + + ": hop-server.sh --host 127.0.0.1 --kill --shutdownport 8082 --userName cluster --password cluster"); } private static void shutdown( @@ -639,18 +798,18 @@ private static void shutdown( try { callStopHopServerRestService(hostname, port, shutdownPort, username, password); } catch (Exception e) { - e.printStackTrace(); + e.printStackTrace(System.err); } } /** * Checks that HopServer is running and if so, shuts down the HopServer server * - * @param hostname - * @param port - * @param username - * @param password - * @throws HopServerCommandException + * @param hostname The hostname + * @param port The port + * @param username The username + * @param password The password + * @throws HopServerCommandException In case there was a command line error */ @VisibleForTesting static void callStopHopServerRestService( diff --git a/engine/src/main/java/org/apache/hop/www/HopServerSingleton.java b/engine/src/main/java/org/apache/hop/www/HopServerSingleton.java index dfd754c057f..28b87c4526c 100644 --- a/engine/src/main/java/org/apache/hop/www/HopServerSingleton.java +++ b/engine/src/main/java/org/apache/hop/www/HopServerSingleton.java @@ -43,11 +43,11 @@ public class HopServerSingleton { - private static final Class PKG = org.apache.hop.www.HopServer.class; + private static final Class PKG = HopServer.class; private static HopServerConfig hopServerConfig; private static HopServerSingleton hopServerSingleton; - private static org.apache.hop.www.HopServer hopServer; + private static HopServer hopServer; private ILogChannel log; @@ -66,16 +66,16 @@ private HopServerSingleton(HopServerConfig config) throws HopException { installPurgeTimer(config, log, pipelineMap, workflowMap); - HopServerMeta hopServer = config.getHopServer(); - if (hopServer != null) { - int port = WebServer.CONST_PORT; - if (!Utils.isEmpty(hopServer.getPort())) { + HopServerMeta hopServerMeta = config.getHopServer(); + if (hopServerMeta != null) { + int port = WebServer.DEFAULT_PORT; + if (!Utils.isEmpty(hopServerMeta.getPort())) { try { - port = Integer.parseInt(hopServer.getPort()); + port = Integer.parseInt(hopServerMeta.getPort()); } catch (Exception e) { log.logError( BaseMessages.getString( - PKG, "HopServer.Error.CanNotPartPort", hopServer.getHostname(), "" + port), + PKG, "HopServer.Error.CanNotPartPort", hopServerMeta.getHostname(), "" + port), e); } } @@ -106,7 +106,8 @@ public static void installPurgeTimer( // if (objectTimeout > 0) { - log.logBasic("Installing timer to purge stale objects after " + objectTimeout + " minutes."); + log.logDetailed( + "Installing timer to purge stale objects after " + objectTimeout + " minutes."); Timer timer = new Timer(true); @@ -217,8 +218,8 @@ public static HopServerSingleton getInstance() { if (hopServerSingleton == null) { if (hopServerConfig == null) { hopServerConfig = new HopServerConfig(); - HopServerMeta hopServer = new HopServerMeta(); - hopServerConfig.setHopServer(hopServer); + HopServerMeta hopServerMeta = new HopServerMeta(); + hopServerConfig.setHopServer(hopServerMeta); } hopServerSingleton = new HopServerSingleton(hopServerConfig); @@ -262,11 +263,11 @@ public static void setHopServerConfig(HopServerConfig hopServerConfig) { HopServerSingleton.hopServerConfig = hopServerConfig; } - public static void setHopServer(org.apache.hop.www.HopServer hopServer) { + public static void setHopServer(HopServer hopServer) { HopServerSingleton.hopServer = hopServer; } - public static org.apache.hop.www.HopServer getHopServer() { + public static HopServer getHopServer() { return HopServerSingleton.hopServer; } diff --git a/engine/src/main/java/org/apache/hop/www/IWebServerShutdownHandler.java b/engine/src/main/java/org/apache/hop/www/IWebServerShutdownHandler.java index b8fa10ff1c0..d19e2099e3b 100644 --- a/engine/src/main/java/org/apache/hop/www/IWebServerShutdownHandler.java +++ b/engine/src/main/java/org/apache/hop/www/IWebServerShutdownHandler.java @@ -6,13 +6,14 @@ * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. + * */ package org.apache.hop.www; diff --git a/engine/src/main/java/org/apache/hop/www/RemoteHopServer.java b/engine/src/main/java/org/apache/hop/www/RemoteHopServer.java new file mode 100644 index 00000000000..c36c3fd8b65 --- /dev/null +++ b/engine/src/main/java/org/apache/hop/www/RemoteHopServer.java @@ -0,0 +1,916 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.www; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static java.util.Objects.requireNonNull; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.net.URLEncoder; +import java.util.HashMap; +import java.util.Map; +import java.util.Random; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import javax.net.ssl.SSLContext; +import org.apache.commons.lang.StringUtils; +import org.apache.hop.core.Const; +import org.apache.hop.core.Result; +import org.apache.hop.core.encryption.Encr; +import org.apache.hop.core.exception.HopException; +import org.apache.hop.core.logging.ILogChannel; +import org.apache.hop.core.logging.LogChannel; +import org.apache.hop.core.util.Utils; +import org.apache.hop.core.variables.IVariables; +import org.apache.hop.core.variables.Variable; +import org.apache.hop.core.vfs.HopVfs; +import org.apache.hop.i18n.BaseMessages; +import org.apache.hop.server.HopServerMeta; +import org.apache.hop.server.ServerConnectionManager; +import org.apache.http.HttpEntity; +import org.apache.http.HttpHost; +import org.apache.http.HttpResponse; +import org.apache.http.HttpStatus; +import org.apache.http.StatusLine; +import org.apache.http.auth.AuthScope; +import org.apache.http.auth.UsernamePasswordCredentials; +import org.apache.http.client.AuthCache; +import org.apache.http.client.ClientProtocolException; +import org.apache.http.client.CredentialsProvider; +import org.apache.http.client.HttpClient; +import org.apache.http.client.config.RequestConfig; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.client.methods.HttpUriRequest; +import org.apache.http.client.protocol.HttpClientContext; +import org.apache.http.conn.ssl.NoopHostnameVerifier; +import org.apache.http.conn.ssl.SSLConnectionSocketFactory; +import org.apache.http.conn.ssl.TrustSelfSignedStrategy; +import org.apache.http.conn.ssl.TrustStrategy; +import org.apache.http.entity.ByteArrayEntity; +import org.apache.http.entity.InputStreamEntity; +import org.apache.http.impl.auth.BasicScheme; +import org.apache.http.impl.client.BasicAuthCache; +import org.apache.http.impl.client.BasicCredentialsProvider; +import org.apache.http.impl.client.HttpClients; +import org.apache.http.message.BasicHeader; +import org.apache.http.ssl.SSLContexts; + +public class RemoteHopServer { + private static final Class PKG = RemoteHopServer.class; + + /** A variable to configure the number of retries for Hop server to send data */ + @Variable( + value = "0", + description = "A variable to configure the number of retries for Hop server to send data") + public static final String HOP_SERVER_RETRIES = "HOP_SERVER_RETRIES"; + + /** to configure the time in millisends to wait before retrying to send data */ + @Variable( + value = "1000", + description = + "A variable to configure the time in millisends to wait before retrying to send data") + public static final String HOP_SERVER_RETRY_BACKOFF_INCREMENTS = + "HOP_SERVER_RETRY_BACKOFF_INCREMENTS"; + + public static final String PROTOCOL_HTTP = "http"; + public static final String PROTOCOL_HTTPS = "https"; + private static final Random RANDOM = new Random(); + private static final String STRING_HOP_SERVER = "Hop Server"; + + private static final String CONST_NAME = "?name="; + private static final String CONST_XML = "&xml=Y"; + private static final String CONST_ID = "&id="; + + private ILogChannel log; + private HopServerMeta serverMeta; + + private static int getNumberOfHopServerRetries(IVariables variables) { + try { + return Integer.parseInt(variables.getVariable(HOP_SERVER_RETRIES, "0")); + } catch (Exception e) { + return 0; + } + } + + public static int getBackoffIncrements(IVariables variables) { + try { + return Integer.parseInt(variables.getVariable(HOP_SERVER_RETRY_BACKOFF_INCREMENTS, "1000")); + } catch (Exception e) { + return 1000; + } + } + + public RemoteHopServer(HopServerMeta serverMeta) { + this.log = new LogChannel(STRING_HOP_SERVER); + this.serverMeta = requireNonNull(serverMeta, "ServerMeta"); + } + + public ILogChannel getLog() { + return log; + } + + public String getName() { + return serverMeta.getName(); + } + + protected String getPortSpecification(IVariables variables) { + String port = variables.resolve(serverMeta.getPort()); + String portSpec = ":" + port; + if (Utils.isEmpty(port) || port.equals("80")) { + portSpec = ""; + } + return portSpec; + } + + public String createUrl(IVariables variables, String serviceAndArguments) { + String hostname = variables.resolve(serverMeta.getHostname()); + String proxyHostname = variables.resolve(serverMeta.getProxyHostname()); + if (!Utils.isEmpty(proxyHostname) && hostname.equals("localhost")) { + hostname = "127.0.0.1"; + } + + if (!StringUtils.isBlank(serverMeta.getWebAppName())) { + serviceAndArguments = + "/" + variables.resolve(serverMeta.getWebAppName()) + serviceAndArguments; + } + + String url = + (serverMeta.isSslMode() ? PROTOCOL_HTTPS : PROTOCOL_HTTP) + + "://" + + hostname + + getPortSpecification(variables) + + serviceAndArguments; + url = Const.replace(url, " ", "%20"); + return url; + } + + HttpPost buildSendXmlMethod(IVariables variables, byte[] content, String service) { + String encoding = Const.XML_ENCODING; + return buildSendMethod(variables, content, encoding, service, "text/xml"); + } + + // Method is defined as package-protected in order to be accessible by unit tests + HttpPost buildSendMethod( + IVariables variables, byte[] content, String encoding, String service, String contentType) { + // Prepare HTTP put + // + String url = createUrl(variables, service); + if (log.isDebug()) { + log.logDebug(BaseMessages.getString(PKG, "HopServer.DEBUG_ConnectingTo", url)); + } + HttpPost method = new HttpPost(url); + + // Request content will be retrieved directly from the input stream + // + HttpEntity entity = new ByteArrayEntity(content); + method.setEntity(entity); + method.addHeader(new BasicHeader("Accept", contentType + ";charset=" + encoding)); + + return method; + } + + public String sendXml(IVariables variables, String xml, String service) throws Exception { + String encoding = getXmlEncoding(xml); + HttpPost method = + buildSendMethod(variables, xml.getBytes(encoding), encoding, service, "text/xml"); + try { + return executeAuth(variables, method); + } finally { + // Release current connection to the connection pool once you are done + method.releaseConnection(); + if (log.isDetailed()) { + log.logDetailed( + BaseMessages.getString( + PKG, + "HopServer.DETAILED_SentXmlToService", + service, + variables.resolve(serverMeta.getHostname()))); + } + } + } + + public String sendJson(IVariables variables, String json, String service) throws Exception { + String encoding = Const.XML_ENCODING; + HttpPost method = + buildSendMethod(variables, json.getBytes(encoding), encoding, service, "application/json"); + try { + return executeAuth(variables, method); + } finally { + // Release current connection to the connection pool once you are done + method.releaseConnection(); + if (log.isDetailed()) { + log.logDetailed( + BaseMessages.getString( + PKG, + "HopServer.DETAILED_SentXmlToService", + service, + variables.resolve(serverMeta.getHostname()))); + } + } + } + + private String getXmlEncoding(String xml) { + Pattern xmlHeadPattern = Pattern.compile("<\\?xml.* encoding=\"(.*)\""); + Matcher matcher = xmlHeadPattern.matcher(xml); + if (matcher.find()) { + return matcher.group(); + } + + return Const.XML_ENCODING; + } + + /** Throws if not ok */ + private void handleStatus( + IVariables variables, HttpUriRequest method, StatusLine statusLine, int status) + throws HopException { + if (status >= 300) { + String message; + if (status == HttpStatus.SC_NOT_FOUND) { + message = + String.format( + "%s%s%s%s", + BaseMessages.getString(PKG, "HopServer.Error.404.Title"), + Const.CR, + Const.CR, + BaseMessages.getString(PKG, "HopServer.Error.404.Message")); + } else { + message = + String.format( + "HTTP Status %d - %s - %s", + status, method.getURI().toString(), statusLine.getReasonPhrase()); + } + throw new HopException(message); + } + } + + // Method is defined as package-protected in order to be accessible by unit tests + HttpPost buildSendExportMethod(IVariables variables, String type, String load, InputStream is) { + String serviceUrl = RegisterPackageServlet.CONTEXT_PATH; + if (type != null && load != null) { + serviceUrl += + "/?" + + RegisterPackageServlet.PARAMETER_TYPE + + "=" + + type + + "&" + + RegisterPackageServlet.PARAMETER_LOAD + + "=" + + URLEncoder.encode(load, UTF_8); + } + + String urlString = createUrl(variables, serviceUrl); + if (log.isDebug()) { + log.logDebug(BaseMessages.getString(PKG, "HopServer.DEBUG_ConnectingTo", urlString)); + } + + HttpPost method = new HttpPost(urlString); + method.setEntity(new InputStreamEntity(is)); + method.addHeader(new BasicHeader("Content-Type", "binary/zip")); + + return method; + } + + /** + * Send an exported archive over to this hop server + * + * @param filename The archive to send + * @param type The type of file to add to the hop server (AddExportServlet.TYPE_*) + * @param load The filename to load in the archive (the .hwf or .hpl) + * @return the XML of the web result + * @throws Exception in case something goes awry + */ + public String sendExport(IVariables variables, String filename, String type, String load) + throws Exception { + // Request content will be retrieved directly from the input stream + try (InputStream is = HopVfs.getInputStream(HopVfs.getFileObject(filename))) { + // Execute request + HttpPost method = buildSendExportMethod(variables, type, load, is); + try { + return executeAuth(variables, method); + } finally { + // Release current connection to the connection pool once you are done + method.releaseConnection(); + if (log.isDetailed()) { + log.logDetailed( + BaseMessages.getString( + PKG, + "HopServer.DETAILED_SentExportToService", + RegisterPackageServlet.CONTEXT_PATH, + variables.resolve(serverMeta.getHostname()))); + } + } + } + } + + /** + * Executes method with authentication. + * + * @param method + * @return + * @throws IOException + * @throws ClientProtocolException + * @throws HopException if response not ok + */ + private String executeAuth(IVariables variables, HttpUriRequest method) + throws IOException, HopException { + HttpResponse httpResponse = getHttpClient().execute(method, getAuthContext(variables)); + return getResponse(variables, method, httpResponse); + } + + private String getResponse(IVariables variables, HttpUriRequest method, HttpResponse httpResponse) + throws IOException, HopException { + StatusLine statusLine = httpResponse.getStatusLine(); + int statusCode = statusLine.getStatusCode(); + // The status code + if (log.isDebug()) { + log.logDebug( + BaseMessages.getString( + PKG, "HopServer.DEBUG_ResponseStatus", Integer.toString(statusCode))); + } + + String responseBody = getResponseBodyAsString(httpResponse.getEntity().getContent()); + if (log.isDebug()) { + log.logDebug(BaseMessages.getString(PKG, "HopServer.DEBUG_ResponseBody", responseBody)); + } + + // throw if not ok + handleStatus(variables, method, statusLine, statusCode); + + return responseBody; + } + + private void addCredentials(IVariables variables, HttpClientContext context) { + + String host = variables.resolve(serverMeta.getHostname()); + int port = Const.toInt(variables.resolve(serverMeta.getPort()), 80); + String userName = variables.resolve(serverMeta.getUsername()); + String password = + Encr.decryptPasswordOptionallyEncrypted(variables.resolve(serverMeta.getPassword())); + String proxyHost = variables.resolve(serverMeta.getProxyHostname()); + + CredentialsProvider provider = new BasicCredentialsProvider(); + UsernamePasswordCredentials credentials = new UsernamePasswordCredentials(userName, password); + if (!Utils.isEmpty(proxyHost) && host.equals("localhost")) { + host = "127.0.0.1"; + } + provider.setCredentials(new AuthScope(host, port), credentials); + context.setCredentialsProvider(provider); + // Generate BASIC scheme object and add it to the local auth cache + HttpHost target = + new HttpHost(host, port, serverMeta.isSslMode() ? PROTOCOL_HTTPS : PROTOCOL_HTTP); + AuthCache authCache = new BasicAuthCache(); + BasicScheme basicAuth = new BasicScheme(); + authCache.put(target, basicAuth); + context.setAuthCache(authCache); + } + + private void addProxy(IVariables variables, HttpClientContext context) { + String proxyHost = variables.resolve(serverMeta.getProxyHostname()); + String proxyPort = variables.resolve(serverMeta.getProxyPort()); + String nonProxyHosts = variables.resolve(serverMeta.getNonProxyHosts()); + + String hostName = variables.resolve(serverMeta.getHostname()); + if (Utils.isEmpty(proxyHost) || Utils.isEmpty(proxyPort)) { + return; + } + // skip applying proxy if non-proxy host matches + if (!Utils.isEmpty(nonProxyHosts) && hostName.matches(nonProxyHosts)) { + return; + } + HttpHost httpHost = new HttpHost(proxyHost, Integer.valueOf(proxyPort)); + + RequestConfig requestConfig = RequestConfig.custom().setProxy(httpHost).build(); + + context.setRequestConfig(requestConfig); + } + + /** + * @return HttpClientContext with authorization credentials + */ + protected HttpClientContext getAuthContext(IVariables variables) { + HttpClientContext context = HttpClientContext.create(); + addCredentials(variables, context); + addProxy(variables, context); + return context; + } + + public String execService(IVariables variables, String service, boolean retry) throws Exception { + int tries = 0; + int maxRetries = 0; + int retryBackoffIncrements = getBackoffIncrements(variables); + if (retry) { + maxRetries = getNumberOfHopServerRetries(variables); + } + while (true) { + try { + return execService(variables, service); + } catch (Exception e) { + if (tries >= maxRetries) { + throw e; + } else { + try { + Thread.sleep(getDelay(tries, retryBackoffIncrements)); + } catch (InterruptedException e2) { + // ignore + } + } + } + tries++; + } + } + + public static long getDelay(int trial, int retryBackoffIncrements) { + long current = retryBackoffIncrements; + long previous = 0; + for (int i = 0; i < trial; i++) { + long tmp = current; + current = current + previous; + previous = tmp; + } + return current + RANDOM.nextInt((int) Math.min(Integer.MAX_VALUE, current / 4L)); + } + + public String execService(IVariables variables, String service) throws Exception { + return execService(variables, service, new HashMap<>()); + } + + // Method is defined as package-protected in order to be accessible by unit tests + protected String getResponseBodyAsString(InputStream is) throws IOException { + BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(is, UTF_8)); + StringBuilder bodyBuffer = new StringBuilder(); + String line; + + try { + while ((line = bufferedReader.readLine()) != null) { + bodyBuffer.append(line); + } + } finally { + bufferedReader.close(); + } + + return bodyBuffer.toString(); + } + + // Method is defined as package-protected in order to be accessible by unit tests + HttpGet buildExecuteServiceMethod( + IVariables variables, String service, Map headerValues) { + HttpGet method = new HttpGet(createUrl(variables, service)); + + for (String key : headerValues.keySet()) { + method.setHeader(key, headerValues.get(key)); + } + return method; + } + + public String execService(IVariables variables, String service, Map headerValues) + throws Exception { + + // Prepare HTTP get + HttpGet method = buildExecuteServiceMethod(variables, service, headerValues); + // Execute request + try { + HttpResponse httpResponse = getHttpClient().execute(method, getAuthContext(variables)); + StatusLine statusLine = httpResponse.getStatusLine(); + int statusCode = statusLine.getStatusCode(); + + // The status code + if (log.isDebug()) { + log.logDebug( + BaseMessages.getString( + PKG, "HopServer.DEBUG_ResponseStatus", Integer.toString(statusCode))); + } + + String responseBody = getResponseBodyAsString(httpResponse.getEntity().getContent()); + + if (log.isDetailed()) { + log.logDetailed( + BaseMessages.getString( + PKG, + "HopServer.DETAILED_FinishedReading", + Integer.toString(responseBody.getBytes().length))); + } + if (log.isDebug()) { + log.logDebug(BaseMessages.getString(PKG, "HopServer.DEBUG_ResponseBody", responseBody)); + } + + if (statusCode >= 400) { + throw new HopException( + String.format( + "HTTP Status %d - %s - %s", + statusCode, method.getURI().toString(), statusLine.getReasonPhrase())); + } + + return responseBody; + } finally { + // Release current connection to the connection pool once you are done + method.releaseConnection(); + if (log.isDetailed()) { + log.logDetailed( + BaseMessages.getString( + PKG, + "HopServer.DETAILED_ExecutedService", + service, + variables.resolve(serverMeta.getHostname()))); + } + } + } + + // Method is defined as package-protected in order to be accessible by unit tests + HttpClient getHttpClient() throws HopException { + try { + if (serverMeta.isSslMode()) { + // Connect over an HTTPS connection + // + TrustStrategy acceptingTrustStrategy = new TrustSelfSignedStrategy(); + SSLContext sslContext = + SSLContexts.custom().loadTrustMaterial(null, acceptingTrustStrategy).build(); + + SSLConnectionSocketFactory socketFactory = + new SSLConnectionSocketFactory(sslContext, NoopHostnameVerifier.INSTANCE); + return HttpClients.custom().setSSLSocketFactory(socketFactory).build(); + } else { + // Connect using a regular HTTP connection, use connection manager to limit the number of + // open connections to hop servers. + // + return ServerConnectionManager.getInstance().createHttpClient(); + } + } catch (Exception e) { + throw new HopException("Error creating new HTTP client", e); + } + } + + public HopServerStatus requestServerStatus(IVariables variables) throws Exception { + String xml = execService(variables, GetStatusServlet.CONTEXT_PATH + "/?xml=Y"); + return HopServerStatus.fromXml(xml); + } + + public HopServerPipelineStatus requestPipelineStatus( + IVariables variables, String pipelineName, String serverObjectId, int startLogLineNr) + throws Exception { + return requestPipelineStatus(variables, pipelineName, serverObjectId, startLogLineNr, false); + } + + public HopServerPipelineStatus requestPipelineStatus( + IVariables variables, + String pipelineName, + String serverObjectId, + int startLogLineNr, + boolean sendResultXmlWithStatus) + throws Exception { + String query = + GetPipelineStatusServlet.CONTEXT_PATH + + CONST_NAME + + URLEncoder.encode(pipelineName, UTF_8) + + CONST_ID + + Const.NVL(serverObjectId, "") + + "&xml=Y&from=" + + startLogLineNr; + if (sendResultXmlWithStatus) { + query = query + "&" + GetPipelineStatusServlet.SEND_RESULT + "=Y"; + } + String xml = execService(variables, query, true); + return HopServerPipelineStatus.fromXml(xml); + } + + public HopServerWorkflowStatus requestWorkflowStatus( + IVariables variables, String workflowName, String serverObjectId, int startLogLineNr) + throws Exception { + String xml = + execService( + variables, + GetWorkflowStatusServlet.CONTEXT_PATH + + CONST_NAME + + URLEncoder.encode(workflowName, UTF_8) + + CONST_ID + + Const.NVL(serverObjectId, "") + + "&xml=Y&from=" + + startLogLineNr, + true); + return HopServerWorkflowStatus.fromXml(xml); + } + + public WebResult requestStopPipeline( + IVariables variables, String pipelineName, String serverObjectId) throws Exception { + String xml = + execService( + variables, + StopPipelineServlet.CONTEXT_PATH + + CONST_NAME + + URLEncoder.encode(pipelineName, UTF_8) + + CONST_ID + + Const.NVL(serverObjectId, "") + + CONST_XML); + return WebResult.fromXmlString(xml); + } + + public WebResult requestPauseResumePipeline( + IVariables variables, String pipelineName, String serverObjectId) throws Exception { + String xml = + execService( + variables, + PausePipelineServlet.CONTEXT_PATH + + CONST_NAME + + URLEncoder.encode(pipelineName, UTF_8) + + CONST_ID + + Const.NVL(serverObjectId, "") + + CONST_XML); + return WebResult.fromXmlString(xml); + } + + public WebResult requestRemovePipeline( + IVariables variables, String pipelineName, String serverObjectId) throws Exception { + String xml = + execService( + variables, + RemovePipelineServlet.CONTEXT_PATH + + CONST_NAME + + URLEncoder.encode(pipelineName, UTF_8) + + CONST_ID + + Const.NVL(serverObjectId, "") + + CONST_XML); + return WebResult.fromXmlString(xml); + } + + public WebResult requestRemoveWorkflow( + IVariables variables, String workflowName, String serverObjectId) throws Exception { + String xml = + execService( + variables, + RemoveWorkflowServlet.CONTEXT_PATH + + CONST_NAME + + URLEncoder.encode(workflowName, UTF_8) + + CONST_ID + + Const.NVL(serverObjectId, "") + + CONST_XML); + return WebResult.fromXmlString(xml); + } + + public WebResult requestStopWorkflow( + IVariables variables, String pipelineName, String serverObjectId) throws Exception { + String xml = + execService( + variables, + StopWorkflowServlet.CONTEXT_PATH + + CONST_NAME + + URLEncoder.encode(pipelineName, UTF_8) + + "&xml=Y&id=" + + Const.NVL(serverObjectId, "")); + return WebResult.fromXmlString(xml); + } + + public WebResult requestStartPipeline( + IVariables variables, String pipelineName, String serverObjectId) throws Exception { + String xml = + execService( + variables, + StartPipelineServlet.CONTEXT_PATH + + CONST_NAME + + URLEncoder.encode(pipelineName, UTF_8) + + CONST_ID + + Const.NVL(serverObjectId, "") + + CONST_XML); + return WebResult.fromXmlString(xml); + } + + public WebResult requestStartWorkflow( + IVariables variables, String workflowName, String serverObjectId) throws Exception { + String xml = + execService( + variables, + StartWorkflowServlet.CONTEXT_PATH + + CONST_NAME + + URLEncoder.encode(workflowName, UTF_8) + + "&xml=Y&id=" + + Const.NVL(serverObjectId, "")); + return WebResult.fromXmlString(xml); + } + + public WebResult requestShutdownServer(IVariables variables) throws Exception { + execService(variables, ShutdownServlet.CONTEXT_PATH); + return WebResult.OK; + } + + /** + * Sniff rows on a the hop server, return xml containing the row metadata and data. + * + * @param pipelineName pipeline name + * @param id the id on the server + * @param transformName transform name + * @param copyNr transform copy number + * @param lines lines number + * @param type transform type + * @return xml with row metadata and data + * @throws Exception + */ + public String sniffTransform( + IVariables variables, + String pipelineName, + String transformName, + String id, + String copyNr, + int lines, + String type) + throws Exception { + return execService( + variables, + SniffTransformServlet.CONTEXT_PATH + + "?pipeline=" + + URLEncoder.encode(pipelineName, UTF_8) + + CONST_ID + + URLEncoder.encode(id, UTF_8) + + "&transform=" + + URLEncoder.encode(transformName, UTF_8) + + "©nr=" + + copyNr + + "&type=" + + type + + "&lines=" + + lines + + CONST_XML); + } + + /** + * Monitors a remote pipeline every 5 seconds. + * + * @param log the log channel interface + * @param serverObjectId the HopServer object ID + * @param pipelineName the pipeline name + */ + public void monitorRemotePipeline( + IVariables variables, ILogChannel log, String serverObjectId, String pipelineName) { + monitorRemotePipeline(variables, log, serverObjectId, pipelineName, 5); + } + + /** + * Monitors a remote pipeline at the specified interval. + * + * @param log the log channel interface + * @param serverObjectId the HopServer object ID + * @param pipelineName the pipeline name + * @param sleepTimeSeconds the sleep time (in seconds) + */ + public void monitorRemotePipeline( + IVariables variables, + ILogChannel log, + String serverObjectId, + String pipelineName, + int sleepTimeSeconds) { + long errors = 0; + boolean allFinished = false; + while (!allFinished && errors == 0) { + allFinished = true; + errors = 0L; + + // Check the remote server + if (allFinished && errors == 0) { + try { + HopServerPipelineStatus pipelineStatus = + requestPipelineStatus(variables, pipelineName, serverObjectId, 0); + if (pipelineStatus.isRunning()) { + if (log.isDetailed()) { + log.logDetailed(pipelineName, "Remote pipeline is still running."); + } + allFinished = false; + } else { + if (log.isDetailed()) { + log.logDetailed(pipelineName, "Remote pipeline has finished."); + } + } + Result result = pipelineStatus.getResult(); + errors += result.getNrErrors(); + } catch (Exception e) { + errors += 1; + log.logError( + pipelineName, + "Unable to contact remote hop server '" + + variables.resolve(serverMeta.getName()) + + "' to check pipeline status : " + + e); + } + } + + // + // Keep waiting until all pipelines have finished + // If needed, we stop them again and again until they yield. + // + if (!allFinished) { + // Not finished or error: wait a bit longer + if (log.isDetailed()) { + log.logDetailed( + pipelineName, "The remote pipeline is still running, waiting a few seconds..."); + } + try { + Thread.sleep(sleepTimeSeconds * 1000L); + } catch (Exception e) { + // Ignore errors + } + } + } + + log.logBasic(pipelineName, "The remote pipeline has finished."); + } + + /** + * Monitors a remote workflow every 5 seconds. + * + * @param log the log channel interface + * @param serverObjectId the HopServer object ID + * @param workflowName the workflow name + */ + public void monitorRemoteWorkflow( + IVariables variables, ILogChannel log, String serverObjectId, String workflowName) { + monitorRemoteWorkflow(variables, log, serverObjectId, workflowName, 5); + } + + /** + * Monitors a remote workflow at the specified interval. + * + * @param log the log channel interface + * @param serverObjectId the HopServer object ID + * @param workflowName the workflow name + * @param sleepTimeSeconds the sleep time (in seconds) + */ + public void monitorRemoteWorkflow( + IVariables variables, + ILogChannel log, + String serverObjectId, + String workflowName, + int sleepTimeSeconds) { + long errors = 0; + boolean allFinished = false; + while (!allFinished && errors == 0) { + allFinished = true; + errors = 0L; + + // Check the remote server + if (allFinished && errors == 0) { + try { + HopServerWorkflowStatus workflowStatus = + requestWorkflowStatus(variables, workflowName, serverObjectId, 0); + if (workflowStatus.isRunning()) { + if (log.isDetailed()) { + log.logDetailed(workflowName, "Remote workflow is still running."); + } + allFinished = false; + } else { + if (log.isDetailed()) { + log.logDetailed(workflowName, "Remote workflow has finished."); + } + } + Result result = workflowStatus.getResult(); + errors += result.getNrErrors(); + } catch (Exception e) { + errors += 1; + log.logError( + workflowName, + "Unable to contact remote hop server '" + + variables.resolve(serverMeta.getName()) + + "' to check workflow status : " + + e); + } + } + + // + // Keep waiting until all pipelines have finished + // If needed, we stop them again and again until they yield. + // + if (!allFinished) { + // Not finished or error: wait a bit longer + if (log.isDetailed()) { + log.logDetailed( + workflowName, "The remote workflow is still running, waiting a few seconds..."); + } + try { + Thread.sleep(sleepTimeSeconds * 1000L); + } catch (Exception e) { + // Ignore errors + } + } + } + + log.logBasic(workflowName, "The remote workflow has finished."); + } + + public HopServerMeta getServerMeta() { + return serverMeta; + } +} diff --git a/engine/src/main/java/org/apache/hop/www/ShutdownServlet.java b/engine/src/main/java/org/apache/hop/www/ShutdownServlet.java new file mode 100644 index 00000000000..c2d8ae4bf99 --- /dev/null +++ b/engine/src/main/java/org/apache/hop/www/ShutdownServlet.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hop.www; + +import jakarta.servlet.ServletException; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; +import java.io.IOException; +import java.io.Serial; +import java.util.Timer; +import java.util.TimerTask; +import org.apache.hop.core.annotations.HopServerServlet; + +@HopServerServlet(id = "shutdown", name = "Shut down the server") +public class ShutdownServlet extends BaseHttpServlet implements IHopServerPlugin { + + @Serial private static final long serialVersionUID = -911533231051351L; + + public static final String CONTEXT_PATH = "/shutdown"; + + @Override + public void doGet(HttpServletRequest request, HttpServletResponse response) + throws ServletException, IOException { + + logBasic( + "Received server shutdown from user '" + + request.getRemoteUser() + + "' at address " + + request.getRemoteAddr()); + + TimerTask task = + new TimerTask() { + public void run() { + // Gracefully shutdown the web server + HopServer server = HopServerSingleton.getHopServer(); + if (server != null) { + server.shutdown(); + } + } + }; + + // Delay shutdown to give servlet time to respond + new Timer("Server shutdown timer").schedule(task, 2000L); + response.setStatus(HttpServletResponse.SC_OK); + } + + public String toString() { + return "Shut down the server"; + } + + @Override + public String getService() { + return CONTEXT_PATH + " (" + toString() + ")"; + } + + @Override + public String getContextPath() { + return CONTEXT_PATH; + } +} diff --git a/engine/src/main/java/org/apache/hop/www/WebServer.java b/engine/src/main/java/org/apache/hop/www/WebServer.java index a3c90552b39..6cc2ce80962 100644 --- a/engine/src/main/java/org/apache/hop/www/WebServer.java +++ b/engine/src/main/java/org/apache/hop/www/WebServer.java @@ -19,11 +19,6 @@ import jakarta.servlet.Servlet; import java.awt.GraphicsEnvironment; -import java.io.BufferedReader; -import java.io.InputStreamReader; -import java.net.InetAddress; -import java.net.ServerSocket; -import java.net.Socket; import java.util.List; import lombok.Getter; import lombok.Setter; @@ -73,33 +68,34 @@ import org.eclipse.jetty.util.ssl.SslContextFactory; import org.glassfish.jersey.servlet.ServletContainer; +@Getter +@Setter public class WebServer { + private static final Class PKG = WebServer.class; - public static final int CONST_PORT = 80; - public static final int SHUTDOWN_PORT = 8079; + public static final int DEFAULT_PORT = 80; + public static final int DEFAULT_SHUTDOWN_PORT = 8079; private static final int DEFAULT_DETECTION_TIMER = 20000; - private static final Class PKG = WebServer.class; + public static final String CONST_WEB_SERVER_LOG_CONFIG_OPTIONS = "WebServer.Log.ConfigOptions"; - @Getter @Setter private ILogChannel log; + private ILogChannel log; /** value of variables */ - @Setter @Getter private IVariables variables; + private IVariables variables; - @Getter @Setter private Server server; - @Getter @Setter private PipelineMap pipelineMap; - @Setter @Getter private WorkflowMap workflowMap; + private Server server; + private PipelineMap pipelineMap; + private WorkflowMap workflowMap; /** the hostname */ - @Setter @Getter private String hostname; + private String hostname; - @Setter @Getter private int port; + private int port; private final int shutdownPort; - private String passwordFile; private final WebServerShutdownHook webServerShutdownHook; /** Can be used to override the default shutdown behavior of performing a System.exit */ - @Setter private IWebServerShutdownHandler webServerShutdownHandler = new DefaultWebServerShutdownHandler(); @@ -164,17 +160,6 @@ public WebServer( } } - public WebServer( - ILogChannel log, - PipelineMap pipelineMap, - WorkflowMap workflowMap, - String hostname, - int port, - int shutdownPort) - throws Exception { - this(log, pipelineMap, workflowMap, hostname, port, shutdownPort, true); - } - public WebServer( ILogChannel log, PipelineMap pipelineMap, @@ -263,6 +248,8 @@ public void startServer() throws Exception { } private ContextHandlerCollection createContexts() throws HopPluginException { + // Configure the Servlet Context, to add all the server plugins + // ContextHandlerCollection contexts = new ContextHandlerCollection(); // Root @@ -273,6 +260,7 @@ private ContextHandlerCollection createContexts() throws HopPluginException { rootServlet.setJettyMode(true); boolean graphicsEnvironment = supportGraphicEnvironment(); + PluginRegistry pluginRegistry = PluginRegistry.getInstance(); List plugins = pluginRegistry.getPlugins(HopServerPluginType.class); for (IPlugin plugin : plugins) { @@ -401,7 +389,7 @@ private ServerConnector getConnector() { /** * Set up jetty options to the connector * - * @param connector + * @param connector The Jetty connector to set up */ protected void setupJettyOptions(ServerConnector connector) { LowResourceMonitor lowResourceMonitor = new LowResourceMonitor(server); @@ -460,14 +448,6 @@ private boolean validProperty(String property) { return isValid; } - public String getPasswordFile() { - return passwordFile; - } - - public void setPasswordFile(String passwordFile) { - this.passwordFile = passwordFile; - } - public int defaultDetectionTimer() { String sDetectionTimer = System.getProperty(Const.HOP_SERVER_DETECTION_TIMER); @@ -485,36 +465,4 @@ private boolean supportGraphicEnvironment() { } return false; } - - private static class MonitorThread extends Thread { - - private final ServerSocket socket; - private final Server server; - - public MonitorThread(Server server, String hostname, int shutdownPort) { - this.server = server; - setDaemon(true); - setName("StopMonitor"); - try { - socket = new ServerSocket(shutdownPort, 1, InetAddress.getByName(hostname)); - } catch (Exception e) { - throw new RuntimeException(e); - } - } - - @Override - public void run() { - Socket accept; - try { - accept = socket.accept(); - BufferedReader reader = new BufferedReader(new InputStreamReader(accept.getInputStream())); - reader.readLine(); - server.stop(); - accept.close(); - socket.close(); - } catch (Exception e) { - throw new RuntimeException(e); - } - } - } } diff --git a/engine/src/main/resources/org/apache/hop/www/messages/messages_en_US.properties b/engine/src/main/resources/org/apache/hop/www/messages/messages_en_US.properties index 3b16538f1b5..1d1b284120d 100644 --- a/engine/src/main/resources/org/apache/hop/www/messages/messages_en_US.properties +++ b/engine/src/main/resources/org/apache/hop/www/messages/messages_en_US.properties @@ -88,6 +88,9 @@ HopServer.Error.CanNotPartShutdownPort=Unable to parse shutdown port [{0}] HopServer.Error.NoServerFound="No Hop Server Server found at {0}\:{1} HopServer.Error.NoShutdown=Cannot shut down hopServer at {0}\:{1} HopServer.Error.illegalStop=Cannot use configuration file to shutdown Hop Server server +HopServer.Log.StartingServer=Starting the Hop server +HopServer.Log.StoppedServer=Hop server is stopped +HopServer.Log.ShuttingDown=Shutdown signal received HopServer.Usage.Example=Example HopServerStatusServlet.BackToHopServerStatus=Back to Hop Server status HopServerStatusServlet.CleanupPipeline=Cleanup pipeline @@ -165,15 +168,16 @@ StopPipelineServlet.StopPipeline=Stop pipeline StopWorkflowServlet.Log.CoundNotFindWorkflow=Workflow [{0}] could not be found. StopWorkflowServlet.log.StopWorkflowRequested=Stop of workflow requested WebResult.Error.UnableCreateResult=Unable to create webresult from XML -WebServer.Error.FailedToStop.Msg=failed to stop webserver \: {0} +WebServer.Error.FailedToStop.Msg=Failed to stop webserver \: {0} WebServer.Error.FailedToStop.Title=web server error WebServer.Error.IllegalSslParameter={0} cannot be {1}. WebServer.Log.ConfigOptions=Configuration option [{0}] set up in {1} WebServer.Log.ConfigOptionsInvalid=Invalid configuration option [{0}]\: {1}. Using default server value. -WebServer.Log.CreateListener=Created listener for webserver @ address \: {0}\:{1} +WebServer.Log.CreateListener=Created listener for [{0}] @ address \: {1}\:{2} WebServer.Log.CreateShutDownListener=Created Shutdown listener for webserver @ address \: {0}\:{1} WebServer.Log.HopHTTPListener=Hop HTTP listener for [{0}] -WebServer.Log.ShuttingDown=Shutdown signal received +WebServer.Log.StartingServer=Starting the web server +WebServer.Log.StoppingServer=Stopping the web server WebServer.Log.SslModeUsing=Using SSL mode WebService.description=Allows you to run a pipeline to generate output for a servlet on Hop Server WebService.name=Web Service diff --git a/engine/src/main/resources/org/apache/hop/www/messages/messages_es_ES.properties b/engine/src/main/resources/org/apache/hop/www/messages/messages_es_ES.properties index b38ad69659b..a5c3c0f31ee 100644 --- a/engine/src/main/resources/org/apache/hop/www/messages/messages_es_ES.properties +++ b/engine/src/main/resources/org/apache/hop/www/messages/messages_es_ES.properties @@ -169,7 +169,7 @@ WebServer.Error.FailedToStop.Title=error del servidor web WebServer.Error.IllegalSslParameter={0} no puede ser {1}. WebServer.Log.ConfigOptions=Opci\u00F3n de configuraci\u00F3n [{0}] establecida en {1} WebServer.Log.ConfigOptionsInvalid=Opci\u00F3n de configuraci\u00F3n no v\u00E1lida [{0}]\: {1}. Usando valor por defecto del servidor. -WebServer.Log.CreateListener=Creada escucha para servidor web @ direcci\u00F3n \: {0}\:{1} +WebServer.Log.CreateListener=Creada escucha para servidor [{0}] @ direcci\u00F3n \: {1}\:{2} WebServer.Log.HopHTTPListener=Hop HTTP listener para [{0}] WebServer.Log.SslModeUsing=Usando el modo SSL WebServiceServlet.Log.WebServiceRequested=Servicio web solicitado diff --git a/engine/src/main/resources/org/apache/hop/www/messages/messages_fr_FR.properties b/engine/src/main/resources/org/apache/hop/www/messages/messages_fr_FR.properties index 695a95a9285..079eb0bdad3 100644 --- a/engine/src/main/resources/org/apache/hop/www/messages/messages_fr_FR.properties +++ b/engine/src/main/resources/org/apache/hop/www/messages/messages_fr_FR.properties @@ -172,7 +172,7 @@ WebServer.Error.FailedToStop.Title=Erreur service web WebServer.Error.IllegalSslParameter={0} ne peut pas \u00EAtre {1}\u00A0. WebServer.Log.ConfigOptions=Option de configuration [{0}] d\u00E9finie sur {1} WebServer.Log.ConfigOptionsInvalid=Option de configuration invalide [{0}]\u202F: {1}. Utilisation de la valeur par d\u00E9faut du serveur\u00A0. -WebServer.Log.CreateListener=Ecoute du service sur @ addresse \: {0}\:{1} +WebServer.Log.CreateListener=Ecoute du service pour [{0}] sur @ addresse \: {1}\:{2} WebServer.Log.HopHTTPListener=Ecoute HTTP de [{0}] WebServer.Log.SslModeUsing=Utilisation du mode SSL WebServiceServlet.Log.WebServiceRequested=Service Web requis diff --git a/engine/src/main/resources/org/apache/hop/www/messages/messages_it_IT.properties b/engine/src/main/resources/org/apache/hop/www/messages/messages_it_IT.properties index 9ae617fe5b6..76d2e3f1b55 100644 --- a/engine/src/main/resources/org/apache/hop/www/messages/messages_it_IT.properties +++ b/engine/src/main/resources/org/apache/hop/www/messages/messages_it_IT.properties @@ -112,7 +112,7 @@ StopWorkflowServlet.log.StopWorkflowRequested=Richiesto l''arresto del workflow WebResult.Error.UnableCreateResult=Impossibile creare il risultato web da XML WebServer.Error.FailedToStop.Msg=fallito l''arresto del server web\: {0} WebServer.Error.FailedToStop.Title=Errore server Web -WebServer.Log.CreateListener=Creato il listener per il server web all''indirizzo\: {0}\:{1} +WebServer.Log.CreateListener=Creato il listener per il server [{0}] all''indirizzo\: {1}\:{2} WebServer.Log.HopHTTPListener=HTTP Listener di Hop per [{0}] WorkflowStatusServlet.BackToWorkflowStatusPage=Indietro alla pagina dello stato del workflow WorkflowStatusServlet.Log.WorkflowStopRequested=Richiesto stop per il workflow [{0}] diff --git a/engine/src/main/resources/org/apache/hop/www/messages/messages_ja_JP.properties b/engine/src/main/resources/org/apache/hop/www/messages/messages_ja_JP.properties index af04078c114..5594ed9ab0c 100644 --- a/engine/src/main/resources/org/apache/hop/www/messages/messages_ja_JP.properties +++ b/engine/src/main/resources/org/apache/hop/www/messages/messages_ja_JP.properties @@ -111,7 +111,7 @@ StopWorkflowServlet.log.StopWorkflowRequested=\u8981\u6c42\u3055\u308c\u305f\u30 WebResult.Error.UnableCreateResult=XML\u304b\u3089webresult\u3092\u4f5c\u6210\u3067\u304d\u307e\u305b\u3093\u3002 WebServer.Error.FailedToStop.Msg=\u30a6\u30a7\u30d6\u30b5\u30fc\u30d0\u304c\u505c\u6b62\u3067\u304d\u307e\u305b\u3093 \: {0} WebServer.Error.FailedToStop.Title=\u30a6\u30a7\u30d6\u30b5\u30fc\u30d0\u30a8\u30e9\u30fc -WebServer.Log.CreateListener=\u30a6\u30a7\u30d6\u30b5\u30fc\u30d0\u30fc @\u30a2\u30c9\u30ec\u30b9 \: {0}\:{1}\u306b\u5bfe\u3057\u3066\u751f\u6210\u3055\u308c\u305f\u30ea\u30b9\u30ca\u30fc +WebServer.Log.CreateListener=\u30a6\u30a7\u30d6\u30b5\u30fc\u30d0\u30fc @\u30a2\u30c9\u30ec\u30b9 [{0}] \: {1}\:{2}\u306b\u5bfe\u3057\u3066\u751f\u6210\u3055\u308c\u305f\u30ea\u30b9\u30ca\u30fc WebServer.Log.HopHTTPListener=[{0}]\u306b\u5bfe\u3059\u308bHop HTTP \u30ea\u30b9\u30ca\u30fc WorkflowStatusServlet.BackToWorkflowStatusPage=\u30b8\u30e7\u30d6\u30b9\u30c6\u30fc\u30bf\u30b9\u30da\u30fc\u30b8\u3078\u623b\u308b WorkflowStatusServlet.Log.WorkflowStopRequested=\u30b8\u30e7\u30d6 [{0}] \u306e\u505c\u6b62\u8981\u6c42 diff --git a/engine/src/main/resources/org/apache/hop/www/messages/messages_ko_KR.properties b/engine/src/main/resources/org/apache/hop/www/messages/messages_ko_KR.properties index a3ddeb818a6..be7e56932fe 100644 --- a/engine/src/main/resources/org/apache/hop/www/messages/messages_ko_KR.properties +++ b/engine/src/main/resources/org/apache/hop/www/messages/messages_ko_KR.properties @@ -97,7 +97,7 @@ StopWorkflowServlet.log.StopWorkflowRequested=Workflow \uC911\uC9C0 \uC694\uCCAD WebResult.Error.UnableCreateResult=XML\uC5D0\uC11C \uACB0\uACFC\uB97C \uC0DD\uC131\uD560 \uC218 \uC5C6\uC2B5\uB2C8\uB2E4 WebServer.Error.FailedToStop.Msg=\uC6F9\uC11C\uBC84 \uC911\uC9C0 \uC2E4\uD328 \: {0} WebServer.Error.FailedToStop.Title=\uC6F9\uC11C\uBC84 \uC624\uB958 -WebServer.Log.CreateListener=\uC6F9\uC11C\uBC84 \uB9AC\uC2A4\uD130 \uC0DD\uC131 @ \uC8FC\uC18C \: {0}\:{1} +WebServer.Log.CreateListener=\uC6F9\uC11C\uBC84 \uB9AC\uC2A4\uD130 \uC0DD\uC131 @ \uC8FC\uC18C [{0}] \: {1}\:{2} WebServer.Log.HopHTTPListener=Hop HTTP listener for [{0}] WorkflowStatusServlet.BackToWorkflowStatusPage=Workflow \uC0C1\uD0DC \uD398\uC774\uC9C0\uB85C \uB3CC\uC544\uAC00\uAE30 WorkflowStatusServlet.Log.WorkflowStopRequested=Workflow [{0}] \uC911\uC9C0 \uC694\uCCAD. diff --git a/engine/src/main/resources/org/apache/hop/www/messages/messages_zh_CN.properties b/engine/src/main/resources/org/apache/hop/www/messages/messages_zh_CN.properties index 5e2a25ce2d4..dbf13d12707 100644 --- a/engine/src/main/resources/org/apache/hop/www/messages/messages_zh_CN.properties +++ b/engine/src/main/resources/org/apache/hop/www/messages/messages_zh_CN.properties @@ -90,6 +90,7 @@ HopServer.Error.CanNotPartShutdownPort=\u65E0\u6CD5\u89E3\u6790\u5173\u95ED\u7AE HopServer.Error.illegalStop=\u5173\u95ED\u7684 Hop Server \u65E0\u6CD5\u4F7F\u7528\u914D\u7F6E\u6587\u4EF6 HopServer.Error.NoServerFound=\u65E0\u6CD5\u8FDE\u63A5 {0}\:{1} \u4E0A\u7684 Hop Server HopServer.Error.NoShutdown=\u65E0\u6CD5\u5173\u95ED Hop Server {0}\:{1} +HopServer.Log.ShuttingDown=\u6536\u5230\u5173\u95ED\u4FE1\u53F7 HopServer.Usage.Example=\u6837\u4F8B HopServerStatusServlet.BackToHopServerStatus=\u8FD4\u56DE Hop Server \u72B6\u6001\u9875\u9762 HopServerStatusServlet.CleanupPipeline=\u6E05\u7406 Pipeline @@ -173,9 +174,8 @@ WebServer.Error.IllegalSslParameter={0} \u4E0D\u80FD\u4E3A {1} WebServer.Log.ConfigOptions=\u914D\u7F6E\u9009\u9879 [{0}] \u8BBE\u7F6E\u5728 {1} \u4E2D WebServer.Log.ConfigOptionsInvalid=\u65E0\u6548\u7684\u914D\u7F6E\u9009\u9879 [{0}]\: {1}, \u4F7F\u7528\u670D\u52A1\u5668\u9ED8\u8BA4\u914D\u7F6E. WebServer.Log.CreateListener=\u521B\u5EFA web \u670D\u52A1\u76D1\u542C\u5668 @ \u5730\u5740\: {0}\:{1} -WebServer.Log.CreateShutDownListener=\u4E3AWeb\u670D\u52A1\u5668\u521B\u5EFA\u4E86\u5173\u95ED\u76D1\u542C\u5668 @ \u5730\u5740 : {0}:{1} +WebServer.Log.CreateShutDownListener=\u4E3AWeb\u670D\u52A1\u5668\u521B\u5EFA\u4E86\u5173\u95ED\u76D1\u542C\u5668 [{0}] @ \u5730\u5740 : {1}:{2} WebServer.Log.HopHTTPListener=\ [{0}]\u7684 Hop HTTP \u76D1\u542C\u5668 -WebServer.Log.ShuttingDown=\u6536\u5230\u5173\u95ED\u4FE1\u53F7 WebServer.Log.SslModeUsing=SSL \u6A21\u5F0F WebService.description=\u5141\u8BB8\u60A8\u8FD0\u884C\u7BA1\u9053\u4EE5\u751F\u6210Hop Server\u4E0AServlet\u7684\u8F93\u51FA WebService.name=Web \u670D\u52A1 diff --git a/engine/src/test/java/org/apache/hop/server/HopServerMetaTest.java b/engine/src/test/java/org/apache/hop/server/HopServerMetaTest.java new file mode 100644 index 00000000000..5250891f4cf --- /dev/null +++ b/engine/src/test/java/org/apache/hop/server/HopServerMetaTest.java @@ -0,0 +1,91 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.server; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.mockito.Mockito.spy; + +import java.util.ArrayList; +import java.util.List; +import org.apache.hop.core.Const; +import org.apache.hop.core.encryption.Encr; +import org.apache.hop.core.encryption.TwoWayPasswordEncoderPluginType; +import org.apache.hop.core.exception.HopException; +import org.apache.hop.core.plugins.PluginRegistry; +import org.apache.hop.core.util.EnvUtil; +import org.apache.hop.core.variables.IVariables; +import org.apache.hop.utils.TestUtils; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + +/** Tests for HopServerMeta class */ +public class HopServerMetaTest { + HopServerMeta hopServer; + IVariables variables; + + @BeforeClass + public static void beforeClass() throws HopException { + PluginRegistry.addPluginType(TwoWayPasswordEncoderPluginType.getInstance()); + PluginRegistry.init(); + String passwordEncoderPluginID = + Const.NVL(EnvUtil.getSystemProperty(Const.HOP_PASSWORD_ENCODER_PLUGIN), "Hop"); + Encr.init(passwordEncoderPluginID); + } + + @AfterClass + public static void tearDown() { + PluginRegistry.getInstance().reset(); + } + + @Test + public void testModifyingName() { + HopServerMeta hopServer1 = spy(new HopServerMeta()); + hopServer1.setName("test"); + List list = new ArrayList<>(); + list.add(hopServer1); + + HopServerMeta hopServer2 = spy(new HopServerMeta()); + hopServer2.setName("test"); + + hopServer2.verifyAndModifyHopServerName(list, null); + + assertFalse(hopServer1.getName().equals(hopServer2.getName())); + } + + @Test + public void testEqualsHashCodeConsistency() { + HopServerMeta server = new HopServerMeta(); + server.setName("server"); + TestUtils.checkEqualsHashCodeConsistency(server, server); + + HopServerMeta serverSame = new HopServerMeta(); + serverSame.setName("server"); + assertEquals(server, serverSame); + TestUtils.checkEqualsHashCodeConsistency(server, serverSame); + + HopServerMeta serverCaps = new HopServerMeta(); + serverCaps.setName("SERVER"); + TestUtils.checkEqualsHashCodeConsistency(server, serverCaps); + + HopServerMeta serverOther = new HopServerMeta(); + serverOther.setName("something else"); + TestUtils.checkEqualsHashCodeConsistency(server, serverOther); + } +} diff --git a/engine/src/test/java/org/apache/hop/www/GetWorkflowStatusServletTest.java b/engine/src/test/java/org/apache/hop/www/GetWorkflowStatusServletTest.java index ddd69d81cab..d1aa5dc3d40 100644 --- a/engine/src/test/java/org/apache/hop/www/GetWorkflowStatusServletTest.java +++ b/engine/src/test/java/org/apache/hop/www/GetWorkflowStatusServletTest.java @@ -17,7 +17,7 @@ package org.apache.hop.www; -import static junit.framework.Assert.assertFalse; +import static org.junit.Assert.assertFalse; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.mock; diff --git a/engine/src/test/java/org/apache/hop/www/PausePipelineServletTest.java b/engine/src/test/java/org/apache/hop/www/PausePipelineServletTest.java index 52fdf86e48b..57514bd9878 100644 --- a/engine/src/test/java/org/apache/hop/www/PausePipelineServletTest.java +++ b/engine/src/test/java/org/apache/hop/www/PausePipelineServletTest.java @@ -17,7 +17,7 @@ package org.apache.hop.www; -import static junit.framework.Assert.assertFalse; +import static org.junit.Assert.assertFalse; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.mock; diff --git a/engine/src/test/java/org/apache/hop/www/PrepareExecutionPipelineServletTest.java b/engine/src/test/java/org/apache/hop/www/PrepareExecutionPipelineServletTest.java index c9214e178af..3af808c9430 100644 --- a/engine/src/test/java/org/apache/hop/www/PrepareExecutionPipelineServletTest.java +++ b/engine/src/test/java/org/apache/hop/www/PrepareExecutionPipelineServletTest.java @@ -16,7 +16,7 @@ */ package org.apache.hop.www; -import static junit.framework.Assert.assertFalse; +import static org.junit.Assert.assertFalse; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.mock; diff --git a/engine/src/test/java/org/apache/hop/server/HopServerTest.java b/engine/src/test/java/org/apache/hop/www/RemoteHopServerTest.java similarity index 80% rename from engine/src/test/java/org/apache/hop/server/HopServerTest.java rename to engine/src/test/java/org/apache/hop/www/RemoteHopServerTest.java index 7e1bb6c56e4..53f3b904041 100644 --- a/engine/src/test/java/org/apache/hop/server/HopServerTest.java +++ b/engine/src/test/java/org/apache/hop/www/RemoteHopServerTest.java @@ -15,10 +15,9 @@ * limitations under the License. */ -package org.apache.hop.server; +package org.apache.hop.www; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; import static org.junit.Assert.fail; @@ -37,8 +36,6 @@ import java.io.InputStream; import java.net.URI; import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.List; import org.apache.hop.core.Const; import org.apache.hop.core.encryption.Encr; import org.apache.hop.core.encryption.TwoWayPasswordEncoderPluginType; @@ -47,7 +44,8 @@ import org.apache.hop.core.util.EnvUtil; import org.apache.hop.core.variables.IVariables; import org.apache.hop.core.variables.Variables; -import org.apache.hop.utils.TestUtils; +import org.apache.hop.server.HopServerMeta; +import org.apache.hop.server.ServerConnectionManager; import org.apache.http.HttpEntity; import org.apache.http.HttpHost; import org.apache.http.HttpResponse; @@ -70,12 +68,12 @@ import org.mockito.stubbing.Answer; /** - * Tests for HopServer class + * Tests for RemoteHopServer class * * @see HopServerMeta */ -public class HopServerTest { - HopServerMeta hopServer; +public class RemoteHopServerTest { + RemoteHopServer hopServer; IVariables variables; @BeforeClass @@ -115,7 +113,7 @@ public void init() throws Exception { .when(httpClient) .execute(any(HttpPost.class), nullable(HttpClientContext.class)); - hopServer = spy(new HopServerMeta()); + hopServer = spy(new RemoteHopServer(new HopServerMeta())); variables = new Variables(); doReturn(httpClient).when(hopServer).getHttpClient(); doReturn("response_body").when(hopServer).getResponseBodyAsString(nullable(InputStream.class)); @@ -156,16 +154,16 @@ public void testExecService() throws Exception { doReturn(httpGetMock) .when(hopServer) .buildExecuteServiceMethod(any(IVariables.class), anyString(), anyMap()); - hopServer.setHostname("hostNameStub"); - hopServer.setUsername("userNAmeStub"); + hopServer.getServerMeta().setHostname("hostNameStub"); + hopServer.getServerMeta().setUsername("userNAmeStub"); hopServer.execService(Variables.getADefaultVariableSpace(), nonExistingAppName); fail("Incorrect connection details had been used, but no exception was thrown"); } @Test(expected = HopException.class) public void testSendXml() throws Exception { - hopServer.setHostname("hostNameStub"); - hopServer.setUsername("userNAmeStub"); + hopServer.getServerMeta().setHostname("hostNameStub"); + hopServer.getServerMeta().setUsername("userNAmeStub"); HttpPost httpPostMock = mock(HttpPost.class); URI uriMock = new URI("fake"); doReturn(uriMock).when(httpPostMock).getURI(); @@ -178,8 +176,8 @@ public void testSendXml() throws Exception { @Test(expected = HopException.class) public void testSendExport() throws Exception { - hopServer.setHostname("hostNameStub"); - hopServer.setUsername("userNAmeStub"); + hopServer.getServerMeta().setHostname("hostNameStub"); + hopServer.getServerMeta().setUsername("userNAmeStub"); HttpPost httpPostMock = mock(HttpPost.class); URI uriMock = new URI("fake"); doReturn(uriMock).when(httpPostMock).getURI(); @@ -196,10 +194,10 @@ public void testSendExport() throws Exception { @Test public void testSendExportOk() throws Exception { - hopServer.setUsername("uname"); - hopServer.setPassword("passw"); - hopServer.setHostname("hname"); - hopServer.setPort("1111"); + hopServer.getServerMeta().setUsername("uname"); + hopServer.getServerMeta().setPassword("passw"); + hopServer.getServerMeta().setHostname("hname"); + hopServer.getServerMeta().setPort("1111"); HttpPost httpPostMock = mock(HttpPost.class); URI uriMock = new URI("fake"); final String responseContent = "baah"; @@ -235,13 +233,13 @@ public void testSendExportOk() throws Exception { @Test public void testAddCredentials() { String testUser = "test_username"; - hopServer.setUsername(testUser); + hopServer.getServerMeta().setUsername(testUser); String testPassword = "test_password"; - hopServer.setPassword(testPassword); + hopServer.getServerMeta().setPassword(testPassword); String host = "somehost"; - hopServer.setHostname(host); + hopServer.getServerMeta().setHostname(host); int port = 1000; - hopServer.setPort("" + port); + hopServer.getServerMeta().setPort("" + port); HttpClientContext auth = hopServer.getAuthContext(variables); Credentials cred = auth.getCredentialsProvider().getCredentials(new AuthScope(host, port)); @@ -249,8 +247,8 @@ public void testAddCredentials() { assertEquals(testPassword, cred.getPassword()); String user2 = "user2"; - hopServer.setUsername(user2); - hopServer.setPassword("pass2"); + hopServer.getServerMeta().setUsername(user2); + hopServer.getServerMeta().setPassword("pass2"); auth = hopServer.getAuthContext(variables); cred = auth.getCredentialsProvider().getCredentials(new AuthScope(host, port)); assertEquals(user2, cred.getUserPrincipal().getName()); @@ -258,11 +256,11 @@ public void testAddCredentials() { @Test public void testAuthCredentialsSchemeWithSSL() { - hopServer.setUsername("admin"); - hopServer.setPassword("password"); - hopServer.setHostname("localhost"); - hopServer.setPort("8443"); - hopServer.setSslMode(true); + hopServer.getServerMeta().setUsername("admin"); + hopServer.getServerMeta().setPassword("password"); + hopServer.getServerMeta().setHostname("localhost"); + hopServer.getServerMeta().setPort("8443"); + hopServer.getServerMeta().setSslMode(true); AuthCache cache = hopServer.getAuthContext(variables).getAuthCache(); assertNotNull(cache.get(new HttpHost("localhost", 8443, "https"))); @@ -271,48 +269,14 @@ public void testAuthCredentialsSchemeWithSSL() { @Test public void testAuthCredentialsSchemeWithoutSSL() { - hopServer.setUsername("admin"); - hopServer.setPassword("password"); - hopServer.setHostname("localhost"); - hopServer.setPort("8080"); - hopServer.setSslMode(false); + hopServer.getServerMeta().setUsername("admin"); + hopServer.getServerMeta().setPassword("password"); + hopServer.getServerMeta().setHostname("localhost"); + hopServer.getServerMeta().setPort("8080"); + hopServer.getServerMeta().setSslMode(false); AuthCache cache = hopServer.getAuthContext(variables).getAuthCache(); assertNull(cache.get(new HttpHost("localhost", 8080, "https"))); assertNotNull(cache.get(new HttpHost("localhost", 8080, "http"))); } - - @Test - public void testModifyingName() { - hopServer.setName("test"); - List list = new ArrayList<>(); - list.add(hopServer); - - HopServerMeta hopServer2 = spy(new HopServerMeta()); - hopServer2.setName("test"); - - hopServer2.verifyAndModifyHopServerName(list, null); - - assertFalse(hopServer.getName().equals(hopServer2.getName())); - } - - @Test - public void testEqualsHashCodeConsistency() { - HopServerMeta server = new HopServerMeta(); - server.setName("server"); - TestUtils.checkEqualsHashCodeConsistency(server, server); - - HopServerMeta serverSame = new HopServerMeta(); - serverSame.setName("server"); - assertEquals(server, serverSame); - TestUtils.checkEqualsHashCodeConsistency(server, serverSame); - - HopServerMeta serverCaps = new HopServerMeta(); - serverCaps.setName("SERVER"); - TestUtils.checkEqualsHashCodeConsistency(server, serverCaps); - - HopServerMeta serverOther = new HopServerMeta(); - serverOther.setName("something else"); - TestUtils.checkEqualsHashCodeConsistency(server, serverOther); - } } diff --git a/engine/src/test/java/org/apache/hop/www/RemovePipelineServletTest.java b/engine/src/test/java/org/apache/hop/www/RemovePipelineServletTest.java index a12aa33a685..8e105baa937 100644 --- a/engine/src/test/java/org/apache/hop/www/RemovePipelineServletTest.java +++ b/engine/src/test/java/org/apache/hop/www/RemovePipelineServletTest.java @@ -17,7 +17,7 @@ package org.apache.hop.www; -import static junit.framework.Assert.assertFalse; +import static org.junit.Assert.assertFalse; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.mock; diff --git a/engine/src/test/java/org/apache/hop/www/RemoveWorkflowServletTest.java b/engine/src/test/java/org/apache/hop/www/RemoveWorkflowServletTest.java index 9cf9eb21f58..4b18f53500f 100644 --- a/engine/src/test/java/org/apache/hop/www/RemoveWorkflowServletTest.java +++ b/engine/src/test/java/org/apache/hop/www/RemoveWorkflowServletTest.java @@ -17,7 +17,7 @@ package org.apache.hop.www; -import static junit.framework.Assert.assertFalse; +import static org.junit.Assert.assertFalse; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.mock; diff --git a/engine/src/test/java/org/apache/hop/www/SniffTransformServletTest.java b/engine/src/test/java/org/apache/hop/www/SniffTransformServletTest.java index aa45252b3b2..a4a38056839 100644 --- a/engine/src/test/java/org/apache/hop/www/SniffTransformServletTest.java +++ b/engine/src/test/java/org/apache/hop/www/SniffTransformServletTest.java @@ -17,7 +17,7 @@ package org.apache.hop.www; -import static junit.framework.Assert.assertFalse; +import static org.junit.Assert.assertFalse; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.mock; diff --git a/engine/src/test/java/org/apache/hop/www/StartExecutionPipelineServletTest.java b/engine/src/test/java/org/apache/hop/www/StartExecutionPipelineServletTest.java index 71464b69ca0..3da296c76ac 100644 --- a/engine/src/test/java/org/apache/hop/www/StartExecutionPipelineServletTest.java +++ b/engine/src/test/java/org/apache/hop/www/StartExecutionPipelineServletTest.java @@ -17,7 +17,7 @@ package org.apache.hop.www; -import static junit.framework.Assert.assertFalse; +import static org.junit.Assert.assertFalse; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.mock; diff --git a/engine/src/test/java/org/apache/hop/www/StartPipelineServletTest.java b/engine/src/test/java/org/apache/hop/www/StartPipelineServletTest.java index d2ce78949d8..5613deae62d 100644 --- a/engine/src/test/java/org/apache/hop/www/StartPipelineServletTest.java +++ b/engine/src/test/java/org/apache/hop/www/StartPipelineServletTest.java @@ -17,7 +17,7 @@ package org.apache.hop.www; -import static junit.framework.Assert.assertFalse; +import static org.junit.Assert.assertFalse; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.mock; diff --git a/engine/src/test/java/org/apache/hop/www/StartWorkflowServletTest.java b/engine/src/test/java/org/apache/hop/www/StartWorkflowServletTest.java index e26bcbffcb1..19b046fb0a7 100644 --- a/engine/src/test/java/org/apache/hop/www/StartWorkflowServletTest.java +++ b/engine/src/test/java/org/apache/hop/www/StartWorkflowServletTest.java @@ -17,7 +17,7 @@ package org.apache.hop.www; -import static junit.framework.Assert.assertFalse; +import static org.junit.Assert.assertFalse; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.mock; diff --git a/engine/src/test/java/org/apache/hop/www/StopWorkflowServletTest.java b/engine/src/test/java/org/apache/hop/www/StopWorkflowServletTest.java index 5018250ff2c..7cb0fd31c8c 100644 --- a/engine/src/test/java/org/apache/hop/www/StopWorkflowServletTest.java +++ b/engine/src/test/java/org/apache/hop/www/StopWorkflowServletTest.java @@ -17,7 +17,7 @@ package org.apache.hop.www; -import static junit.framework.Assert.assertFalse; +import static org.junit.Assert.assertFalse; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.mock; diff --git a/engine/src/test/java/org/apache/hop/www/WebServerTest.java b/engine/src/test/java/org/apache/hop/www/WebServerTest.java index 3fbe992e2ee..6ec2a634db7 100644 --- a/engine/src/test/java/org/apache/hop/www/WebServerTest.java +++ b/engine/src/test/java/org/apache/hop/www/WebServerTest.java @@ -17,6 +17,7 @@ package org.apache.hop.www; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -38,15 +39,19 @@ public class WebServerTest { @ClassRule public static RestoreHopEngineEnvironment env = new RestoreHopEngineEnvironment(); /** */ - private static final String EMPTY_STRING = ""; + private static final String PUBLIC_CONNECTOR_NAME = "webserver"; + + private static final String SHUTDOWN_CONNECTOR_NAME = "shutdown"; - private static final boolean SHOULD_JOIN = false; + private static final String EMPTY_STRING = ""; private static final String HOST_NAME = "localhost"; - private static final int PORT = 8099; + private static final int PORT = 8199; - private static final int SHUTDOEN_PORT = 8098; + private static final int SHUTDOWN_PORT = 8198; + + private static final boolean JOIN = true; private static final String ACCEPTORS = "5"; @@ -58,14 +63,12 @@ public class WebServerTest { private static final int EXPECTED_RES_MAX_IDLE_TIME = 200; - private static final int EXPECTED_CONNECTORS_SIZE = 1; + private static final int EXPECTED_CONNECTORS_SIZE = 2; - private WebServer webServer; - private WebServer webServerNg; - private PipelineMap trMapMock = mock(PipelineMap.class); - private HopServerConfig sServerConfMock = mock(HopServerConfig.class); - private HopServerMeta sServer = mock(HopServerMeta.class); - private WorkflowMap jbMapMock = mock(WorkflowMap.class); + private PipelineMap pipelineMapMock = mock(PipelineMap.class); + private HopServerConfig serverConfigMock = mock(HopServerConfig.class); + private HopServerMeta serverMeta = mock(HopServerMeta.class); + private WorkflowMap workflowMapMock = mock(WorkflowMap.class); private ILogChannel logMock = mock(ILogChannel.class); @Before @@ -74,87 +77,108 @@ public void setup() throws Exception { System.setProperty(Const.HOP_SERVER_JETTY_ACCEPT_QUEUE_SIZE, ACCEPT_QUEUE_SIZE); System.setProperty(Const.HOP_SERVER_JETTY_RES_MAX_IDLE_TIME, RES_MAX_IDLE_TIME); - when(sServerConfMock.getHopServer()).thenReturn(sServer); - when(trMapMock.getHopServerConfig()).thenReturn(sServerConfMock); - when(sServer.getPassword()).thenReturn("cluster"); - when(sServer.getUsername()).thenReturn("cluster"); - webServer = - new WebServer( - logMock, trMapMock, jbMapMock, HOST_NAME, PORT, SHUTDOEN_PORT, SHOULD_JOIN, null); + when(serverConfigMock.getHopServer()).thenReturn(serverMeta); + when(pipelineMapMock.getHopServerConfig()).thenReturn(serverConfigMock); + when(serverMeta.getPassword()).thenReturn("cluster"); + when(serverMeta.getUsername()).thenReturn("cluster"); } @After public void tearDown() { - webServer.setWebServerShutdownHandler(null); // disable system.exit - webServer.stopServer(); - System.getProperties().remove(Const.HOP_SERVER_JETTY_ACCEPTORS); System.getProperties().remove(Const.HOP_SERVER_JETTY_ACCEPT_QUEUE_SIZE); System.getProperties().remove(Const.HOP_SERVER_JETTY_RES_MAX_IDLE_TIME); } @Test - public void testJettyOption_AcceptQueueSizeSetUp() { - assertEquals(EXPECTED_CONNECTORS_SIZE, getSocketConnectors(webServer).size()); - for (ServerConnector sc : getSocketConnectors(webServer)) { - assertEquals(EXPECTED_ACCEPT_QUEUE_SIZE, sc.getAcceptQueueSize()); - } + public void testSocketConnectors() throws Exception { + WebServer webserver = + new WebServer( + logMock, pipelineMapMock, workflowMapMock, HOST_NAME, PORT, SHUTDOWN_PORT, JOIN); + assertEquals(EXPECTED_CONNECTORS_SIZE, getSocketConnectors(webserver).size()); + webserver.stopServer(); + } + + @Test + public void testShutdownDisabled() throws Exception { + WebServer webserver = + new WebServer(logMock, pipelineMapMock, workflowMapMock, HOST_NAME, PORT, -1, JOIN); + assertEquals(1, getSocketConnectors(webserver).size()); + webserver.stopServer(); } @Test - public void testJettyOption_LowResourceMaxIdleTimeSetUp() { - assertEquals(EXPECTED_CONNECTORS_SIZE, getSocketConnectors(webServer).size()); - for (ServerConnector sc : getSocketConnectors(webServer)) { - assertEquals(EXPECTED_RES_MAX_IDLE_TIME, sc.getIdleTimeout()); + public void testJettyOption() throws Exception { + WebServer webserver = + new WebServer(logMock, pipelineMapMock, workflowMapMock, HOST_NAME, PORT, -1, JOIN); + + // AcceptQueueSizeSetUp + for (ServerConnector connector : getSocketConnectors(webserver, PUBLIC_CONNECTOR_NAME)) { + assertEquals(EXPECTED_ACCEPT_QUEUE_SIZE, connector.getAcceptQueueSize()); + } + for (ServerConnector connector : getSocketConnectors(webserver, SHUTDOWN_CONNECTOR_NAME)) { + assertEquals(1, connector.getAcceptQueueSize()); + } + + // LowResourceMaxIdleTimeSetUp + for (ServerConnector connector : getSocketConnectors(webserver, PUBLIC_CONNECTOR_NAME)) { + assertEquals(EXPECTED_RES_MAX_IDLE_TIME, connector.getIdleTimeout()); } + + webserver.stopServer(); } @Test public void testNoExceptionAndUsingDefaultServerValue_WhenJettyOptionSetAsInvalidValue() throws Exception { System.setProperty(Const.HOP_SERVER_JETTY_ACCEPTORS, "TEST"); + WebServer webserver = null; try { - webServerNg = + webserver = new WebServer( - logMock, trMapMock, jbMapMock, HOST_NAME, PORT + 1, SHUTDOEN_PORT, SHOULD_JOIN, null); + logMock, pipelineMapMock, workflowMapMock, HOST_NAME, PORT, SHUTDOWN_PORT, JOIN); } catch (NumberFormatException nmbfExc) { fail("Should not have thrown any NumberFormatException but it does: " + nmbfExc); } - assertEquals(EXPECTED_CONNECTORS_SIZE, getSocketConnectors(webServerNg).size()); - for (ServerConnector sc : getSocketConnectors(webServerNg)) { - assertEquals(sc.getAcceptors(), sc.getAcceptors()); - } - webServerNg.setWebServerShutdownHandler(null); // disable system.exit - webServerNg.stopServer(); + assertTrue(webserver.getServer().isStarted()); + webserver.stopServer(); } @Test public void testNoExceptionAndUsingDefaultServerValue_WhenJettyOptionSetAsEmpty() throws Exception { System.setProperty(Const.HOP_SERVER_JETTY_ACCEPTORS, EMPTY_STRING); + WebServer webserver = null; try { - webServerNg = + webserver = new WebServer( - logMock, trMapMock, jbMapMock, HOST_NAME, PORT + 1, SHUTDOEN_PORT, SHOULD_JOIN, null); + logMock, pipelineMapMock, workflowMapMock, HOST_NAME, PORT, SHUTDOWN_PORT, JOIN); } catch (NumberFormatException nmbfExc) { fail("Should not have thrown any NumberFormatException but it does: " + nmbfExc); } - assertEquals(EXPECTED_CONNECTORS_SIZE, getSocketConnectors(webServerNg).size()); - for (ServerConnector sc : getSocketConnectors(webServerNg)) { - assertEquals(sc.getAcceptors(), sc.getAcceptors()); + assertTrue(webserver.getServer().isStarted()); + webserver.stopServer(); + } + + private List getSocketConnectors(WebServer webserver, String name) { + List connectors = new ArrayList<>(); + for (Connector connector : webserver.getServer().getConnectors()) { + if (connector instanceof ServerConnector serverConnector) { + if (name.equals(serverConnector.getName())) { + connectors.add(serverConnector); + } + } } - webServerNg.setWebServerShutdownHandler(null); // disable system.exit - webServerNg.stopServer(); + return connectors; } - private List getSocketConnectors(WebServer wServer) { - List sConnectors = new ArrayList<>(); - Connector[] connectors = wServer.getServer().getConnectors(); - for (Connector cn : connectors) { - if (cn instanceof ServerConnector) { - sConnectors.add((ServerConnector) cn); + private List getSocketConnectors(WebServer webserver) { + List connectors = new ArrayList<>(); + for (Connector connector : webserver.getServer().getConnectors()) { + if (connector instanceof ServerConnector serverConnector) { + connectors.add(serverConnector); } } - return sConnectors; + return connectors; } } diff --git a/plugins/transforms/serverstatus/src/main/java/org/apache/hop/pipeline/transforms/serverstatus/GetServerStatus.java b/plugins/transforms/serverstatus/src/main/java/org/apache/hop/pipeline/transforms/serverstatus/GetServerStatus.java index d2fc0c8085c..24f54ae219d 100644 --- a/plugins/transforms/serverstatus/src/main/java/org/apache/hop/pipeline/transforms/serverstatus/GetServerStatus.java +++ b/plugins/transforms/serverstatus/src/main/java/org/apache/hop/pipeline/transforms/serverstatus/GetServerStatus.java @@ -28,6 +28,7 @@ import org.apache.hop.www.HopServerPipelineStatus; import org.apache.hop.www.HopServerStatus; import org.apache.hop.www.HopServerWorkflowStatus; +import org.apache.hop.www.RemoteHopServer; public class GetServerStatus extends BaseTransform { public GetServerStatus( @@ -63,10 +64,11 @@ public boolean processRow() throws HopException { } String serverName = getInputRowMeta().getString(row, data.serverFieldIndex); - HopServerMeta hopServer = metadataProvider.getSerializer(HopServerMeta.class).load(serverName); - if (hopServer == null) { + HopServerMeta serverMeta = metadataProvider.getSerializer(HopServerMeta.class).load(serverName); + if (serverMeta == null) { throw new HopException("Hop server '" + serverName + "' couldn't be found"); } + RemoteHopServer server = new RemoteHopServer(serverMeta); String errorMessage; String statusDescription = null; @@ -87,7 +89,7 @@ public boolean processRow() throws HopException { try { errorMessage = null; - HopServerStatus status = hopServer.getStatus(this); + HopServerStatus status = server.requestServerStatus(this); statusDescription = status.getStatusDescription(); serverLoad = status.getLoadAvg(); memoryFree = status.getMemoryFree(); diff --git a/ui/src/main/java/org/apache/hop/ui/hopgui/file/workflow/delegates/HopGuiWorkflowRunDelegate.java b/ui/src/main/java/org/apache/hop/ui/hopgui/file/workflow/delegates/HopGuiWorkflowRunDelegate.java index 473da65f89d..f18fb6d2171 100644 --- a/ui/src/main/java/org/apache/hop/ui/hopgui/file/workflow/delegates/HopGuiWorkflowRunDelegate.java +++ b/ui/src/main/java/org/apache/hop/ui/hopgui/file/workflow/delegates/HopGuiWorkflowRunDelegate.java @@ -36,6 +36,7 @@ import org.apache.hop.ui.workflow.dialog.WorkflowExecutionConfigurationDialog; import org.apache.hop.workflow.WorkflowExecutionConfiguration; import org.apache.hop.workflow.WorkflowMeta; +import org.apache.hop.www.RemoteHopServer; import org.eclipse.swt.SWT; import org.eclipse.swt.widgets.Shell; @@ -108,25 +109,28 @@ WorkflowExecutionConfigurationDialog newWorkflowExecutionConfigurationDialog( hopGui.getShell(), executionConfiguration, workflowMeta); } - private static void showSaveJobBeforeRunningDialog(Shell shell) { + private static void showSaveWorkflowBeforeRunningDialog(Shell shell) { MessageBox m = new MessageBox(shell, SWT.OK | SWT.ICON_WARNING); m.setText(BaseMessages.getString(PKG, "WorkflowLog.Dialog.SaveJobBeforeRunning.Title")); m.setMessage(BaseMessages.getString(PKG, "WorkflowLog.Dialog.SaveJobBeforeRunning.Message")); m.open(); } - private void monitorRemoteJob( + private void monitorRemoteWorkflow( final WorkflowMeta workflowMeta, final String serverObjectId, - final HopServerMeta remoteHopServer) { + final HopServerMeta hopServerMeta) { // There is a workflow running in the background. When it finishes log the result on the // console. + + RemoteHopServer server = new RemoteHopServer(hopServerMeta); + // Launch in a separate thread to prevent GUI blocking... // Thread thread = new Thread( () -> - remoteHopServer.monitorRemoteWorkflow( + server.monitorRemoteWorkflow( hopGui.getVariables(), hopGui.getLog(), serverObjectId, @@ -135,10 +139,10 @@ private void monitorRemoteJob( thread.setName( "Monitor remote workflow '" + workflowMeta.getName() - + "', carte object id=" + + "', server object id=" + serverObjectId + ", hop server: " - + remoteHopServer.getName()); + + server.getName()); thread.start(); } diff --git a/ui/src/main/java/org/apache/hop/ui/server/HopServerMetaEditor.java b/ui/src/main/java/org/apache/hop/ui/server/HopServerMetaEditor.java index c137aa11857..0d9e07e8517 100644 --- a/ui/src/main/java/org/apache/hop/ui/server/HopServerMetaEditor.java +++ b/ui/src/main/java/org/apache/hop/ui/server/HopServerMetaEditor.java @@ -31,6 +31,7 @@ import org.apache.hop.ui.core.widget.TextVar; import org.apache.hop.ui.hopgui.HopGui; import org.apache.hop.www.RegisterPipelineServlet; +import org.apache.hop.www.RemoteHopServer; import org.eclipse.swt.SWT; import org.eclipse.swt.custom.CTabFolder; import org.eclipse.swt.custom.CTabItem; @@ -428,8 +429,11 @@ public boolean setFocus() { public void test() { - HopServerMeta server = getMetadata(); - getWidgetsContent(server); + HopServerMeta serverMeta = getMetadata(); + + getWidgetsContent(serverMeta); + + RemoteHopServer server = new RemoteHopServer(serverMeta); try { String xml = ""; @@ -438,7 +442,7 @@ public void test() { String message = BaseMessages.getString(PKG, "HopServer.Replay.Info1") - + server.constructUrl(manager.getVariables(), RegisterPipelineServlet.CONTEXT_PATH) + + server.createUrl(manager.getVariables(), RegisterPipelineServlet.CONTEXT_PATH) + Const.CR + BaseMessages.getString(PKG, "HopServer.Replay.Info2") + Const.CR @@ -460,7 +464,7 @@ public void test() { getShell(), BaseMessages.getString(PKG, "HopServer.ExceptionError"), BaseMessages.getString(PKG, "HopServer.ExceptionUnableGetReplay.Error1") - + server.getHostname() + + getVariables().resolve(serverMeta.getHostname()) + BaseMessages.getString(PKG, "HopServer.ExceptionUnableGetReplay.Error2"), e); } diff --git a/ui/src/main/resources/org/apache/hop/ui/server/messages/messages_en_US.properties b/ui/src/main/resources/org/apache/hop/ui/server/messages/messages_en_US.properties index 21b2acb4e0a..bbb1053c061 100644 --- a/ui/src/main/resources/org/apache/hop/ui/server/messages/messages_en_US.properties +++ b/ui/src/main/resources/org/apache/hop/ui/server/messages/messages_en_US.properties @@ -23,11 +23,11 @@ HopServer.Replay.Info1=Testing reply from server URL: HopServer.Replay.Info2=Using content: HopServer.RetournedXMLInfo=The XML returned is: HopServerDialog.HostIP.Label=Hostname or IP address -HopServerDialog.IgnoreProxyForHosts.Label=Ignore proxy for hosts: regexp | separated: +HopServerDialog.IgnoreProxyForHosts.Label=Ignore proxy for hosts: regexp | separated HopServerDialog.Password.Label=Password HopServerDialog.Port.Label=Port (empty is port 80) -HopServerDialog.ProxyServerName.Label=Proxy server hostname: -HopServerDialog.ProxyServerPort.Label=The proxy server port: +HopServerDialog.ProxyServerName.Label=Proxy server hostname +HopServerDialog.ProxyServerPort.Label=The proxy server port HopServerDialog.ServerName.Label=Server name HopServerDialog.USER_TAB_PROXY=Proxy HopServerDialog.USER_TAB_SERVICE=Service