NUTCH-3130 Address deprecated API usage across Nutch codebase and build

.github/workflows/master-build.yml

@@ -13,7 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-name: master pull request ci
+name: master branch ci
 on:
   push:
     branches: [master]
@@ -89,16 +89,36 @@ jobs:
               - '.github/workflows/*'
       # run if the build configuration or both 'core' and 'plugins' files were changed
       - name: test all
+        id: build_all
         if: ${{ steps.filter.outputs.buildconf == 'true' || ( steps.filter.outputs.core  == 'true' && steps.filter.outputs.plugins  == 'true' ) }}
-        run: ant clean test -buildfile build.xml
+        run: ant clean test -buildfile build.xml | tee build.log
       # run only if 'core' files were changed
       - name: test core
+        id: build_core
         if: ${{ steps.filter.outputs.core == 'true' && steps.filter.outputs.plugins == 'false' && steps.filter.outputs.buildconf == 'false' }}
-        run: ant clean test-core -buildfile build.xml
+        run: ant clean test-core -buildfile build.xml | tee build.log
       # run only if 'plugins' files were changed
       - name: test plugins
+        id: build_plugins
         if: ${{ steps.filter.outputs.plugins == 'true' && steps.filter.outputs.core == 'false' && steps.filter.outputs.buildconf == 'false' }}
-        run: ant clean test-plugins -buildfile build.xml
+        run: ant clean test-plugins -buildfile build.xml | tee build.log
+      # check for deprecation warnings in build output
+      - name: Check for deprecation warnings
+        if: always()
+        run: |
+          if [ -f build.log ]; then
+            if grep -iEq "warning: \[deprecation\]" build.log ; then
+              echo "============================================================="
+              echo "= ❌ Java deprecation warnings detected! Failing the build. ="
+              echo "============================================================="
+              grep -iE "warning: \[deprecation\]" -A 2 build.log
+              exit 1
+            else
+              echo "✅ No Java deprecation warnings found."
+            fi
+          else
+            echo "⚠️ build.log not found, skipping deprecation check."
+          fi
       - name: Upload Test Report
         uses: actions/upload-artifact@v4
         if: always()
@@ -108,4 +128,4 @@ jobs:
             ./build/test/TEST-*.xml
             ./build/**/test/TEST-*.xml
           retention-days: 1
-          overwrite: true
+          overwrite: true

build.xml

@@ -40,7 +40,7 @@
 
   <property environment="env"/>
 
-  <property name="spotbugs.version" value="4.2.0" />
+  <property name="spotbugs.version" value="4.9.6" />
   <property name="spotbugs.home" value="${ivy.dir}/spotbugs-${spotbugs.version}" />
   <property name="spotbugs.jar" value="${spotbugs.home}/lib/spotbugs-ant.jar" />
 
@@ -79,7 +79,12 @@
   </path>
 
   <presetdef name="javac">
-    <javac includeantruntime="false" />
+    <javac includeantruntime="false" 
+      encoding="${build.encoding}"
+      debug="${javac.debug}"
+      optimize="${javac.optimize}"
+      release="${javac.version}"
+      deprecation="${javac.deprecation}"/>
   </presetdef>
 
   <target name="dependencytree" depends="resolve-default" description="Show dependency tree">
@@ -120,15 +125,9 @@
 
   <target name="compile-core" depends="init, resolve-default" description="--> compile core Java files only">
     <javac
-     encoding="${build.encoding}"
      srcdir="${src.dir}"
      includes="org/apache/nutch/**/*.java"
-     destdir="${build.classes}"
-     debug="${javac.debug}"
-     optimize="${javac.optimize}"
-     target="${javac.version}"
-     source="${javac.version}"
-     deprecation="${javac.deprecation}">
+     destdir="${build.classes}">
       <compilerarg value="-Xlint:-path"/>
       <classpath refid="classpath"/>
     </javac>
@@ -450,15 +449,9 @@
   <!-- ================================================================== -->
   <target name="compile-core-test" depends="init, compile-core, resolve-test" description="--> compile test code">
     <javac
-     encoding="${build.encoding}"
      srcdir="${test.src.dir}"
      includes="org/apache/nutch/**/*.java"
-     destdir="${test.build.classes}"
-     debug="${javac.debug}"
-     optimize="${javac.optimize}"
-     target="${javac.version}"
-     source="${javac.version}"
-     deprecation="${javac.deprecation}">
+     destdir="${test.build.classes}">
       <compilerarg value="-Xlint:-path"/>
       <classpath refid="test.classpath"/>
     </javac>

src/java/org/apache/nutch/crawl/CrawlDbReader.java

@@ -16,47 +16,27 @@
  */
 package org.apache.nutch.crawl;
 
-import java.io.Closeable;
-import java.io.DataOutputStream;
-import java.io.File;
-import java.io.IOException;
-import java.lang.invoke.MethodHandles;
-import java.net.MalformedURLException;
-import java.net.URL;
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.Random;
-import java.util.TreeMap;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
+import com.fasterxml.jackson.core.JsonGenerationException;
+import com.fasterxml.jackson.core.JsonGenerator;
+import com.fasterxml.jackson.core.json.JsonWriteFeature;
+import com.fasterxml.jackson.core.util.MinimalPrettyPrinter;
+import com.fasterxml.jackson.databind.JsonSerializer;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.ObjectWriter;
+import com.fasterxml.jackson.databind.SerializerProvider;
+import com.fasterxml.jackson.databind.module.SimpleModule;
+import com.tdunning.math.stats.MergingDigest;
+import com.tdunning.math.stats.TDigest;
 import org.apache.commons.jexl3.JexlScript;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.BytesWritable;
-import org.apache.hadoop.io.FloatWritable;
-import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.io.MapFile;
-import org.apache.hadoop.io.SequenceFile;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.io.*;
 import org.apache.hadoop.io.compress.CompressionCodec;
 import org.apache.hadoop.io.compress.GzipCodec;
-import org.apache.hadoop.mapreduce.Job;
-import org.apache.hadoop.mapreduce.Mapper;
-import org.apache.hadoop.mapreduce.RecordWriter;
-import org.apache.hadoop.mapreduce.Reducer;
-import org.apache.hadoop.mapreduce.TaskAttemptContext;
+import org.apache.hadoop.mapreduce.*;
 import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
 import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat;
 import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
@@ -67,26 +47,22 @@
 import org.apache.hadoop.util.ReflectionUtils;
 import org.apache.hadoop.util.StringUtils;
 import org.apache.hadoop.util.ToolRunner;
-import org.apache.nutch.util.AbstractChecker;
-import org.apache.nutch.util.JexlUtil;
-import org.apache.nutch.util.NutchConfiguration;
-import org.apache.nutch.util.NutchJob;
-import org.apache.nutch.util.SegmentReaderUtil;
-import org.apache.nutch.util.StringUtil;
-import org.apache.nutch.util.TimingUtil;
+import org.apache.nutch.util.*;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import com.fasterxml.jackson.core.JsonGenerationException;
-import com.fasterxml.jackson.core.JsonGenerator;
-import com.fasterxml.jackson.core.util.MinimalPrettyPrinter;
-import com.fasterxml.jackson.databind.JsonSerializer;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.ObjectWriter;
-import com.fasterxml.jackson.databind.SerializerProvider;
-import com.fasterxml.jackson.databind.module.SimpleModule;
-import com.tdunning.math.stats.MergingDigest;
-import com.tdunning.math.stats.TDigest;
+import java.io.Closeable;
+import java.io.DataOutputStream;
+import java.io.File;
+import java.io.IOException;
+import java.lang.invoke.MethodHandles;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.nio.ByteBuffer;
+import java.util.*;
+import java.util.Map.Entry;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 
 /**
  * Read utility for the CrawlDB.
@@ -263,7 +239,7 @@ protected static class LineRecordWriter
       public LineRecordWriter(DataOutputStream out) {
         this.out = out;
         jsonMapper.getFactory()
-            .configure(JsonGenerator.Feature.ESCAPE_NON_ASCII, true);
+            .configure(JsonWriteFeature.ESCAPE_NON_ASCII.mappedFeature(), true);
         SimpleModule module = new SimpleModule();
         module.addSerializer(Writable.class, new WritableSerializer());
         jsonMapper.registerModule(module);

src/java/org/apache/nutch/indexer/IndexWriter.java

@@ -30,15 +30,6 @@ public interface IndexWriter extends Pluggable, Configurable {
    */
   final static String X_POINT_ID = IndexWriter.class.getName();
 
-  /**
-   * @param conf Nutch configuration
-   * @param name target name of the {@link IndexWriter} to be opened
-   * @throws IOException Some exception thrown by some writer.
-   * @deprecated use {@link #open(IndexWriterParams)}} instead.  
-   */
-  @Deprecated
-  public void open(Configuration conf, String name) throws IOException;
-
   /**
    * Initializes the internal variables from a given index writer configuration.
    *

src/java/org/apache/nutch/indexer/IndexWriters.java

@@ -211,7 +211,7 @@ private Collection<String> getIndexWriters(NutchDocument doc) {
   public void open(Configuration conf, String name) throws IOException {
     for (Map.Entry<String, IndexWriterWrapper> entry : this.indexWriters
         .entrySet()) {
-      entry.getValue().getIndexWriter().open(conf, name);
+      entry.getValue().getIndexWriter().open(new IndexWriterParams(new HashMap<>()));
       entry.getValue().getIndexWriter()
           .open(entry.getValue().getIndexWriterConfig().getParams());
     }

src/java/org/apache/nutch/metadata/SpellCheckedMetadata.java

@@ -101,7 +101,7 @@ private static String normalize(final String str) {
    * <li>CoNtEntType gives Content-Type</li>
    * <li>ConTnTtYpe gives Content-Type</li>
    * </ul>
-   * If no matching with a well-known metadata name is found, then the original
+   * If no well-known metadata name match is found, then the original
    * name is returned.
    * 
    * @param name
@@ -115,7 +115,7 @@ public static String getNormalizedName(final String name) {
     if ((value == null) && (normalized != null)) {
       int threshold = Math.min(3, searched.length() / TRESHOLD_DIVIDER);
       for (int i = 0; i < normalized.length && value == null; i++) {
-        if (StringUtils.getLevenshteinDistance(searched, normalized[i]) < threshold) {
+        if (StringUtils.compareIgnoreCase(searched, normalized[i]) < threshold) { //.getLevenshteinDistance(searched, normalized[i]) < threshold) {
           value = NAMES_IDX.get(normalized[i]);
         }
       }

src/java/org/apache/nutch/plugin/Plugin.java

@@ -88,9 +88,4 @@ public PluginDescriptor getDescriptor() {
   private void setDescriptor(PluginDescriptor descriptor) {
     fDescriptor = descriptor;
   }
-
-  @Override
-  protected void finalize() throws Throwable {
-    shutDown();
-  }
 }

src/java/org/apache/nutch/plugin/PluginRepository.java

@@ -18,6 +18,7 @@
 
 import java.lang.invoke.MethodHandles;
 import java.lang.reflect.Array;
+import java.lang.ref.Cleaner;
 import java.lang.reflect.Constructor;
 import java.lang.reflect.InvocationTargetException;
 import java.lang.reflect.Method;
@@ -70,6 +71,8 @@ public class PluginRepository implements URLStreamHandlerFactory {
 
   protected static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
 
+  private static final Cleaner CLEANER = Cleaner.create();
+
   /**
    * @param conf a populated {@link Configuration}
    * @throws RuntimeException if a fatal runtime error is encountered 
@@ -98,13 +101,22 @@ public PluginRepository(Configuration conf) throws RuntimeException {
     try {
       installExtensions(this.fRegisteredPlugins);
     } catch (PluginRuntimeException e) {
-      LOG.error("Could not install extensions.", e.toString());
+      LOG.error("Could not install extensions. {}", e.toString());
       throw new RuntimeException(e.getMessage());
     }
 
     registerURLStreamHandlerFactory();
 
     displayStatus();
+
+    // Register cleanup action with Cleaner
+    CLEANER.register(this, () -> {
+      try {
+        shutDownActivatedPlugins();
+      } catch (PluginRuntimeException e) {
+        LOG.error("Error during cleanup of activated plugins", e);
+      }
+    });
   }
 
   /**
@@ -313,19 +325,6 @@ public Plugin getPluginInstance(PluginDescriptor pDescriptor)
     }
   }
 
-  /**
-   * Attempts to shut down all activated plugins.
-   * @deprecated
-   * @see <a href="https://openjdk.java.net/jeps/421">JEP 421: Deprecate Finalization for Removal</a>
-   * @see java.lang.Object#finalize()
-   * @deprecated
-   */
-  @Override
-  @Deprecated
-  public void finalize() throws Throwable {
-    shutDownActivatedPlugins();
-  }
-
   /**
    * Shuts down all plugins
    * 

src/java/org/apache/nutch/scoring/webgraph/LinkDumper.java

@@ -27,10 +27,9 @@
 
 import org.apache.commons.cli.CommandLine;
 import org.apache.commons.cli.CommandLineParser;
-import org.apache.commons.cli.GnuParser;
+import org.apache.commons.cli.DefaultParser;
 import org.apache.commons.cli.HelpFormatter;
 import org.apache.commons.cli.Option;
-import org.apache.commons.cli.OptionBuilder;
 import org.apache.commons.cli.Options;
 import org.apache.commons.lang3.time.StopWatch;
 import org.slf4j.Logger;
@@ -417,18 +416,20 @@ public static void main(String[] args) throws Exception {
   public int run(String[] args) throws Exception {
 
     Options options = new Options();
-    OptionBuilder.withArgName("help");
-    OptionBuilder.withDescription("show this help message");
-    Option helpOpts = OptionBuilder.create("help");
+    Option helpOpts = Option.builder("help")
+        .argName("help")
+        .desc("show this help message")
+        .build();
     options.addOption(helpOpts);
 
-    OptionBuilder.withArgName("webgraphdb");
-    OptionBuilder.hasArg();
-    OptionBuilder.withDescription("the web graph database to use");
-    Option webGraphDbOpts = OptionBuilder.create("webgraphdb");
+    Option webGraphDbOpts = Option.builder("webgraphdb")
+        .argName("webgraphdb")
+        .hasArg()
+        .desc("the web graph database to use")
+        .build();
     options.addOption(webGraphDbOpts);
 
-    CommandLineParser parser = new GnuParser();
+    CommandLineParser parser = new DefaultParser();
     try {
 
       CommandLine line = parser.parse(options, args);