Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fulltext fetcher for IACR eprints #9651

Merged
merged 8 commits into from
Mar 6, 2023
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Add tests and fixes for IACR full text fetcher
  • Loading branch information
SECtim committed Mar 6, 2023
commit e574896c39b486b3ad75a77fe575f94892b79969
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ public class IacrEprintFetcher implements FulltextFetcher, IdBasedFetcher {
private static final Predicate<String> IDENTIFIER_PREDICATE = Pattern.compile("\\d{4}/\\d{3,5}").asPredicate();
private static final String CITATION_URL_PREFIX = "https://eprint.iacr.org/";
private static final String DESCRIPTION_URL_PREFIX = "https://eprint.iacr.org/";
private static final String FULLTEXT_URL_PREFIX = "https://eprint.iacr.org/archive/";
private static final String FULLTEXT_URL_PREFIX = "https://eprint.iacr.org/";
private static final String VERSION_URL_PREFIX = "https://eprint.iacr.org/archive/versions/";

private final ImportFormatPreferences prefs;
Expand Down Expand Up @@ -142,7 +142,7 @@ public Optional<URL> findFullText(BibEntry entry) throws IOException, FetcherExc
String fulltextLinkAsInHtml = getRequiredValueBetween(startOfFulltextLink, ".pdf", descriptiveHtml);
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Alternatively, one could use JSOUP here https://jsoup.org/ (see e.g. SemanticScholar) to operate directly on the HTML dom elements, but as this getValueBeteween is already used here it's fine

// There is an additional "\n href=\"/archive/" we have to remove - and for some reason,
// getRequiredValueBetween refuses to match across the line break.
fulltextLinkAsInHtml = fulltextLinkAsInHtml.replaceFirst(".*href=\"/archive/", "").trim();
fulltextLinkAsInHtml = fulltextLinkAsInHtml.replaceFirst(".*href=\"/", "").trim();
String fulltextLink = FULLTEXT_URL_PREFIX + fulltextLinkAsInHtml + ".pdf";
return Optional.of(new URL(fulltextLink));
}
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
package org.jabref.logic.importer.fetcher;

import java.io.IOException;
import java.net.URL;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
Expand Down Expand Up @@ -36,6 +38,7 @@ public class IacrEprintFetcherTest {

private IacrEprintFetcher fetcher;
private BibEntry abram2017;
private BibEntry abram2017noVersion;
private BibEntry beierle2016;
private BibEntry delgado2017;

Expand All @@ -55,6 +58,17 @@ public void setUp() {
.withField(StandardField.VERSION, "20171124:064527")
.withField(StandardField.YEAR, "2017");

abram2017noVersion = new BibEntry(StandardEntryType.Misc)
.withCitationKey("cryptoeprint:2017/1118")
.withField(StandardField.ABSTRACT, "dummy")
.withField(StandardField.AUTHOR, "Ittai Abraham and Dahlia Malkhi and Kartik Nayak and Ling Ren and Alexander Spiegelman")
.withField(StandardField.DATE, "2017-11-24")
.withField(StandardField.HOWPUBLISHED, "Cryptology ePrint Archive, Paper 2017/1118")
.withField(StandardField.NOTE, "\\url{https://eprint.iacr.org/2017/1118}")
.withField(StandardField.TITLE, "Solida: A Blockchain Protocol Based on Reconfigurable Byzantine Consensus")
.withField(StandardField.URL, "https://eprint.iacr.org/2017/1118")
.withField(StandardField.YEAR, "2017");

beierle2016 = new BibEntry(StandardEntryType.Misc)
.withCitationKey("cryptoeprint:2016/119")
.withField(StandardField.ABSTRACT, "dummy")
Expand Down Expand Up @@ -185,4 +199,34 @@ private static Stream<String> allNonWithdrawnIdsWithOldHtmlFormat() {
ids.removeAll(withdrawnIds);
return ids.stream();
}


@Test
public void getFulltextWithVersion() throws FetcherException, IOException {
Optional<URL> pdfUrl = fetcher.findFullText(abram2017);
assertTrue(pdfUrl.isPresent());
assertEquals("https://eprint.iacr.org/archive/2017/1118/1511505927.pdf", pdfUrl.get().toString());
}

@Test
public void getFulltextWithoutVersion() throws FetcherException, IOException {
Optional<URL> pdfUrl = fetcher.findFullText(abram2017noVersion);
assertTrue(pdfUrl.isPresent());
assertEquals("https://eprint.iacr.org/2017/1118.pdf", pdfUrl.get().toString());
}

@Test
public void getFulltextWithoutUrl() throws FetcherException, IOException {
BibEntry abram2017WithoutUrl = abram2017;
abram2017WithoutUrl.clearField(StandardField.URL);
Optional<URL> pdfUrl = fetcher.findFullText(abram2017WithoutUrl);
assertTrue(pdfUrl.isEmpty());
}

@Test
public void getFulltextWithNonIACRUrl() throws FetcherException, IOException {
BibEntry abram2017WithNonIACRUrl = abram2017;
abram2017WithNonIACRUrl.setField(StandardField.URL, "https://example.com");
assertThrows(FetcherException.class, () -> fetcher.findFullText(abram2017WithNonIACRUrl));
}
}