Minor: Added disregarded protocols from yasserg#446

HHN · Jan 24, 2022 · 8b44efb · 8b44efb
1 parent fdb14e0
commit 8b44efb
Showing 1 changed file with 6 additions and 2 deletions.
diff --git a/crawler4j-core/src/main/java/edu/uci/ics/crawler4j/parser/TikaHtmlParser.java b/crawler4j-core/src/main/java/edu/uci/ics/crawler4j/parser/TikaHtmlParser.java
@@ -126,8 +126,12 @@ private Set<WebURL> getOutgoingUrls(String contextURL, HtmlContentHandler conten
             }
 
             String hrefLoweredCase = href.trim().toLowerCase(Locale.ROOT);
-            if (!hrefLoweredCase.contains("javascript:") &&
-                    !hrefLoweredCase.contains("mailto:") && !hrefLoweredCase.contains("@")) {
+            if (!hrefLoweredCase.contains("about:") && !hrefLoweredCase.contains("tel:") &&
+                    !hrefLoweredCase.contains("data:")  && !hrefLoweredCase.contains("whatsapp:") &&
+                    !hrefLoweredCase.contains("javascript:") && !hrefLoweredCase.contains("viber:") &&
+                    !hrefLoweredCase.contains("sms:") && !hrefLoweredCase.contains("android-app:") &&
+                    !hrefLoweredCase.contains("fb-messenger:") && !hrefLoweredCase.contains("mailto:") &&
+                    !hrefLoweredCase.contains("@") && !hrefLoweredCase.contains("fb-messenger:")) {
                 String url = normalizer.filter(UrlResolver.resolveUrl((contextURL == null) ? "" : contextURL, href));
                 if (url != null) {
                     WebURL webURL = factory.newWebUrl();