Skip to content

NUTCH-2481 HostDatum deltas(previous step statistics) and Metadata expressions #278

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
okedoki wants to merge 10 commits into
base: master
Choose a base branch
from
Open

NUTCH-2481 HostDatum deltas(previous step statistics) and Metadata expressions #278

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
60eb705
fix for NUTCH-2461 generate with maxcount equals 0
okedoki Nov 29, 2017
97083fa
NUTCH-2481. HostDatum deltas expressions
okedoki Jan 17, 2018
b2612a4
Revert "fix for NUTCH-2461 generate with maxcount equals 0"
okedoki Jan 17, 2018
226f4b1
addition for NUTCH-2481 delta expression. Now expressions can return …
okedoki Jan 19, 2018
555bfff
NUTCH-2481. Changed the descprition of hostdb.deltaExpression in the …
okedoki Jan 22, 2018
a898b5c
Merge branch 'master', remote branch 'origin'
okedoki Jan 22, 2018
07cfff4
Merge remote branch 'origin/master'
Jan 25, 2018
8d5b917
Merge remote branch 'origin/master'
okedoki Feb 12, 2018
e84018c
Merge branch 'master' into NUTCH-2481
okedoki Feb 12, 2018
785c354
refactoring for usage of nutch JexlUtil
okedoki Feb 12, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions conf/nutch-default.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2588,6 +2588,15 @@ visit https://wiki.apache.org/nutch/SimilarityScoringFilter-->
</description>
</property>

<property>
<name>hostdb.deltaExpression</name>
<value></value>
<description>
The expression for calculation of the delta statistics, the differences between of value of hostdb after update(currentHostDatum) and the value before(previousHostDatum). The return value in the KeyValuePair(String,Integer) format is written to the metadata of the hostdb.
For example, {return new ("javafx.util.Pair","FetchedDelta", currentHostDatum.fetched - previousHostDatum.fetched);}
</description>
</property>

<!-- publisher properties
Do not forget to add the name of your publisher implementation
in plugin.includes ex- publish-rabbitmq -->
Expand Down
2 changes: 1 addition & 1 deletion src/java/org/apache/nutch/hostdb/UpdateHostDb.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,6 @@
import org.apache.nutch.util.NutchConfiguration;
import org.apache.nutch.util.NutchJob;
import org.apache.nutch.util.TimingUtil;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

Expand All @@ -78,6 +77,7 @@ public class UpdateHostDb extends Configured implements Tool {
public static final String HOSTDB_NUMERIC_FIELDS = "hostdb.numeric.fields";
public static final String HOSTDB_STRING_FIELDS = "hostdb.string.fields";
public static final String HOSTDB_PERCENTILES = "hostdb.percentiles";
public static final String HOSTDB_UPDATEDB_DELTA_EXPRESSION = "hostdb.deltaExpression";

private void updateHostDb(Path hostDb, Path crawlDb, Path topHosts,
boolean checkFailed, boolean checkNew, boolean checkKnown,
Expand Down
13 changes: 7 additions & 6 deletions src/java/org/apache/nutch/hostdb/UpdateHostDbMapper.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,15 +19,14 @@
import java.io.IOException;
import java.lang.invoke.MethodHandles;


import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.io.FloatWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.Mapper;
import org.apache.hadoop.mapred.OutputCollector;
import org.apache.hadoop.mapred.Reporter;

import org.apache.nutch.crawl.CrawlDatum;
import org.apache.nutch.crawl.CrawlDb;
import org.apache.nutch.crawl.NutchWritable;
Expand All @@ -36,7 +35,6 @@
import org.apache.nutch.net.URLNormalizers;
import org.apache.nutch.protocol.ProtocolStatus;
import org.apache.nutch.util.URLUtil;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

Expand All @@ -62,6 +60,7 @@ public class UpdateHostDbMapper
protected URLFilters filters = null;
protected URLNormalizers normalizers = null;

private boolean isDeltaStatisticCalculated = false;
public void close() {}

/**
Expand All @@ -71,7 +70,8 @@ public void configure(JobConf job) {
readingCrawlDb = job.getBoolean("hostdb.reading.crawldb", false);
filter = job.getBoolean(UpdateHostDb.HOSTDB_URL_FILTERING, false);
normalize = job.getBoolean(UpdateHostDb.HOSTDB_URL_NORMALIZING, false);

isDeltaStatisticCalculated = !StringUtils.isEmpty(job.get(UpdateHostDb.HOSTDB_UPDATEDB_DELTA_EXPRESSION));

if (filter)
filters = new URLFilters(job);
if (normalize)
Expand Down Expand Up @@ -212,9 +212,10 @@ public void map(Text key, Writable value,

// If we're also reading CrawlDb entries, reset db_* statistics because
// we're aggregating them from CrawlDB anyway
if (readingCrawlDb) {

if (readingCrawlDb && !isDeltaStatisticCalculated) {
hostDatum.resetStatistics();
}
}

output.collect(key, new NutchWritable(hostDatum));
}
Expand Down
Loading