NUTCH-2522 Bidirectional URL exemption filter

build.xml

@@ -227,6 +227,7 @@
       <packageset dir="${plugins.dir}/urlfilter-domain/src/java"/>
       <packageset dir="${plugins.dir}/urlfilter-domainblacklist/src/java"/>
       <packageset dir="${plugins.dir}/urlfilter-ignoreexempt/src/java"/>
+      <packageset dir="${plugins.dir}/urlfilter-ignoreexemptbidirectional/src/java"/>
       <packageset dir="${plugins.dir}/urlfilter-prefix/src/java"/>
       <packageset dir="${plugins.dir}/urlfilter-regex/src/java"/>
       <packageset dir="${plugins.dir}/urlfilter-suffix/src/java"/>
@@ -684,6 +685,7 @@
       <packageset dir="${plugins.dir}/urlfilter-domain/src/java"/>
       <packageset dir="${plugins.dir}/urlfilter-domainblacklist/src/java"/>
       <packageset dir="${plugins.dir}/urlfilter-ignoreexempt/src/java"/>
+      <packageset dir="${plugins.dir}/urlfilter-ignoreexemptbidirectional/src/java"/>
       <packageset dir="${plugins.dir}/urlfilter-prefix/src/java"/>
       <packageset dir="${plugins.dir}/urlfilter-regex/src/java"/>
       <packageset dir="${plugins.dir}/urlfilter-suffix/src/java"/>
@@ -1122,6 +1124,7 @@
         <source path="${plugins.dir}/urlfilter-domainblacklist/src/java/" />
         <source path="${plugins.dir}/urlfilter-domainblacklist/src/test/" />
         <source path="${plugins.dir}/urlfilter-ignoreexempt/src/java/" />
+        <source path="${plugins.dir}/urlfilter-ignoreexempt-bidirectional/src/java"/>
         <source path="${plugins.dir}/urlfilter-prefix/src/java/" />
         <source path="${plugins.dir}/urlfilter-prefix/src/test/" />
         <source path="${plugins.dir}/urlfilter-regex/src/java/" />

conf/db-ignore-external-exemptions-bidirectional.txt

@@ -0,0 +1,33 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+#
+# Exemption rules to db.ignore.external.links
+
+
+# Format :
+#--------
+# The format is same same as `regex-urlfilter.txt`.
+# Each non-comment, non-blank line contains a regular expression
+# prefixed by '+' or '-'.  The first matching pattern in the file
+# determines whether a URL is exempted or ignored.  If no pattern
+# matches, the URL is ignored.
+
+
+
+# Example 1:
+#----------
+# To exempt urls ending with image extensions, uncomment the below line
+-(www.)

conf/nutch-default.xml

@@ -611,6 +611,14 @@
   </description>
 </property>
 
+ <property>
+  <name>db.ignore.external.exemptions.bidirectional.file</name>
+  <value>db-ignore-external-exemptions-bidirectional.txt</value>
+  <description>
+    This file contains exemption rules used by 'urlfiter-ignoreexempt-bidirectional' plugin
+  </description>
+</property>
+
 <property>
   <name>db.injector.overwrite</name>
   <value>false</value>

src/plugin/build.xml

@@ -90,6 +90,7 @@
     <ant dir="urlfilter-validator" target="deploy"/>
     <!-- urlfilter-ignoreexempt depends on urlfilter-regex, must build after -->
     <ant dir="urlfilter-ignoreexempt" target="deploy"/>
+    <ant dir="urlfilter-ignoreexempt-bidirectional" target="deploy"/>
     <ant dir="urlmeta" target="deploy"/>
     <ant dir="urlnormalizer-ajax" target="deploy"/>
     <ant dir="urlnormalizer-basic" target="deploy"/>
@@ -220,6 +221,7 @@
     <ant dir="urlfilter-domain" target="clean" />
     <ant dir="urlfilter-domainblacklist" target="clean" />
     <ant dir="urlfilter-ignoreexempt" target="clean"/>
+    <ant dir="urlfilter-ignoreexempt-bidirectional" target="clean"/>
     <ant dir="urlfilter-prefix" target="clean"/>
     <ant dir="urlfilter-regex" target="clean"/>
     <ant dir="urlfilter-suffix" target="clean"/>

src/plugin/lib-regex-filter/src/java/org/apache/nutch/urlfilter/api/RegexRule.java

@@ -87,7 +87,7 @@ protected boolean accept() {
    *
    * @return this regex
    */
-  protected String regex() { return regex; }
+  public String regex() { return regex; }
 
   /**
    * Checks if a url matches this rule.
@@ -97,6 +97,15 @@ protected boolean accept() {
    * @return <code>true</code> if the specified url matches this rule, otherwise
    *         <code>false</code>.
    */
-  protected abstract boolean match(String url);
+  public abstract boolean match(String url);
 
+  /**
+   * Replace  if a url matches this rule.
+   * 
+   * @param url
+   *          is the url to check.
+   * @return <code>true</code> if the specified url matches this rule, otherwise
+   *         <code>false</code>.
+   */
+  public abstract String replace(String url, String replacement);
 }

src/plugin/lib-regex-filter/src/java/org/apache/nutch/urlfilter/api/RegexURLFilterBase.java

@@ -69,6 +69,10 @@ public abstract class RegexURLFilterBase implements URLFilter {
 
   /** An array of applicable rules */
   private List<RegexRule> rules;
+
+  protected List<RegexRule> getRules(){
+	return rules;  
+  }
 
   /** The current configuration */
   private Configuration conf;

src/plugin/urlfilter-automaton/src/java/org/apache/nutch/urlfilter/automaton/AutomatonURLFilter.java

@@ -22,12 +22,15 @@
 import java.io.StringReader;
 import java.util.regex.PatternSyntaxException;
 
+
 // Hadoop imports
 import org.apache.hadoop.conf.Configuration;
 
+
 // Automaton imports
 import dk.brics.automaton.RegExp;
 import dk.brics.automaton.RunAutomaton;
+
 import org.apache.nutch.net.*;
 import org.apache.nutch.urlfilter.api.RegexRule;
 import org.apache.nutch.urlfilter.api.RegexURLFilterBase;
@@ -108,9 +111,15 @@ private class Rule extends RegexRule {
       automaton = new RunAutomaton(new RegExp(regex, RegExp.ALL).toAutomaton());
     }
 
-    protected boolean match(String url) {
+    public boolean match(String url) {
       return automaton.run(url);
     }
+
+	@Override
+	public String replace(String url, String replacement) {
+		// TODO Auto-generated method stub
+		return null;
+	}
   }
 
 }

src/plugin/urlfilter-ignoreexempt-bidirectional/build.xml

@@ -0,0 +1,37 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements.  See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License.  You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project name="urlfilter-ignoreexempt-bidirectional" default="jar-core">
+
+  <import file="../build-plugin.xml"/>
+
+  <!-- Build compilation dependencies -->
+  <target name="deps-jar">
+    <ant target="jar" inheritall="false" dir="../lib-regex-filter"/>
+  </target>
+
+  <!-- Add compilation dependencies to classpath -->
+  <path id="plugin.deps">
+    <fileset dir="${nutch.root}/build">
+      <include name="**/lib-regex-filter/*.jar" />
+      <include name="**/urlfilter-regex/*.jar" />
+    </fileset>
+    <pathelement location="${nutch.root}/build/lib-regex-filter/test"/>
+  </path>
+
+
+</project>

src/plugin/urlfilter-ignoreexempt-bidirectional/ivy.xml

@@ -0,0 +1,41 @@
+<?xml version="1.0" ?>
+
+<!--
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+-->
+
+<ivy-module version="1.0">
+  <info organisation="org.apache.nutch" module="${ant.project.name}">
+    <license name="Apache 2.0"/>
+    <ivyauthor name="Apache Nutch Team" url="http://nutch.apache.org"/>
+    <description>
+        Apache Nutch
+    </description>
+  </info>
+
+  <configurations>
+    <include file="../../..//ivy/ivy-configurations.xml"/>
+  </configurations>
+
+  <publications>
+    <!--get the artifact from our module name-->
+    <artifact conf="master"/>
+  </publications>
+
+  <dependencies>
+  </dependencies>
+
+</ivy-module>

src/plugin/urlfilter-ignoreexempt-bidirectional/plugin.xml

@@ -0,0 +1,45 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements.  See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License.  You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<plugin
+   id="urlfilter-ignoreexempt-bidirectional"
+   name="External Domain Ignore Exemption Bidirectional"
+   version="1.0.0"
+   provider-name="nutch.org">
+
+   <runtime>
+      <library name="urlfilter-ignoreexemptbidirectional.jar">
+         <export name="*"/>
+      </library>
+   </runtime>
+
+   <requires>
+      <import plugin="nutch-extensionpoints"/>
+      <import plugin="lib-regex-filter"/>
+      <import plugin="urlfilter-regex"/>
+   </requires>
+
+   <extension id="org.apache.nutch.urlfilter.ignoreexempt.bidirectional"
+              name="Ignore Exemption Url Filter Bidirectional"
+              point="org.apache.nutch.net.URLExemptionFilter">
+      <implementation id="BidirectionalExemptionUrlFilter"
+        class="org.apache.nutch.urlfilter.ignoreexempt.bidirectional.BidirectionalExemptionUrlFilter">
+        <parameter name="file" value="db-ignore-external-exemptions-bidirectional.txt"/>
+      </implementation>
+   </extension>
+
+</plugin>

src/plugin/urlfilter-ignoreexempt-bidirectional/src/java/org/apache/nutch/urlfilter/ignoreexempt/bidirectional/BidirectionalExemptionUrlFilter.java

@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nutch.urlfilter.ignoreexempt.bidirectional;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.nutch.net.URLExemptionFilter;
+import org.apache.nutch.util.NutchConfiguration;
+import org.apache.nutch.util.URLUtil;
+import org.apache.nutch.urlfilter.api.RegexRule;
+import org.apache.nutch.urlfilter.regex.RegexURLFilter;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.lang.invoke.MethodHandles;
+import java.net.MalformedURLException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.Reader;
+import java.util.Arrays;
+import java.util.regex.Pattern;
+import java.util.List;
+import java.util.ArrayList;
+
+
+/**
+ * This implementation of {@link org.apache.nutch.net.URLExemptionFilter} uses regex configuration for both fromUrl and toUrl
+ * to check if URL is eligible for exemption from 'db.ignore.external'.
+ * When this filter is enabled, the external urls will be checked against configured sequence of regex rules.
+ *<p>
+ * The exemption rule file defaults to db-ignore-external-exemptions-bidirectional.txt in the classpath but can be
+ * overridden using the property  <code>"db.ignore.external.exemptions.bidirectional.file" in ./conf/nutch-*.xml</code>
+ *</p>
+ * @since Mar 1, 2018
+ * @version 1
+ * @see org.apache.nutch.net.URLExemptionFilter
+ * @see org.apache.nutch.urlfilter.regex.RegexURLFilter
+ */
+public class BidirectionalExemptionUrlFilter extends RegexURLFilter
+    implements URLExemptionFilter {
+
+  public static final String DB_IGNORE_EXTERNAL_EXEMPTIONS_BIDIRECTIONAL_FILE
+      = "db.ignore.external.exemptions.bidirectional.file";
+  private static final Logger LOG = LoggerFactory
+      .getLogger(MethodHandles.lookup().lookupClass());
+
+  @Override
+  //This implementation checks rules exceptions for two arbitrary urls. True if reg_ex(toUrl) = fromUrl
+  public boolean filter(String fromUrl, String toUrl) {
+
+	    String sourceHost = URLUtil.getHost(fromUrl).toLowerCase();
+	    String sourceDestination = URLUtil.getHost(toUrl).toLowerCase();
+
+	    if (LOG.isDebugEnabled()) {
+	      LOG.debug("BidirectionalExemptionUrlFilter. Source url: " + fromUrl + " and destination url " + toUrl);
+	    }
+
+	    String modifiedSourceHost = sourceHost; 
+	    String modifiedDestinationHost = sourceDestination;
+	    for (RegexRule rule : super.getRules()) {
+
+	      if (LOG.isDebugEnabled()) {
+	        LOG.debug("Applying rule [" + rule.regex() + "]");
+	      }
+
+	      modifiedSourceHost = rule.replace(modifiedSourceHost, "");
+	      modifiedDestinationHost = rule.replace(modifiedDestinationHost, "");
+	    };
+
+	    return modifiedSourceHost.equals(modifiedDestinationHost);
+  }
+
+
+  protected Reader getRulesReader(Configuration conf)
+      throws IOException {
+    String fileRules = conf.get(DB_IGNORE_EXTERNAL_EXEMPTIONS_BIDIRECTIONAL_FILE);
+    this.getConf();
+    return conf.getConfResourceAsReader(fileRules);
+  }
+  public static void main(String[] args) {
+
+    if (args.length != 2) {
+      System.out.println("Error: Invalid Args");
+      System.out.println("Usage: " +
+          BidirectionalExemptionUrlFilter.class.getName() + " <url_source, url_destination>");
+      return;
+    }
+    String sourceUrl = args[0];
+    String destinationUrl = args[1];
+    BidirectionalExemptionUrlFilter instance = new BidirectionalExemptionUrlFilter();
+    instance.setConf(NutchConfiguration.create());
+    System.out.println(instance.filter(sourceUrl, destinationUrl));
+  }
+}