Java: RegEx: WebSpider

In this tutorial we will create a basis for a WebSpider program that fetches all links from the given Web page, checks which have extensions of interest and downloads these files.

The result of the app will be a folder with Google Earth map overlays:

/Users/uki/Desktop/KMZ

├── 1.kmz

├── 10.kmz

├── 11.kmz

├── 12.kmz

├── 13.kmz

├── 14.kmz

├── 15.kmz

├── 16.kmz

├── 17.kmz

├── 18.kmz

We will use basic java networking classes:

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.net.MalformedURLException;
import java.net.URL;

and regular expressions:

ublic class RegExConstants {

   /**    * @param extensions    * @return    *  (                -- start of main grouping    *     [^\s]+        -- must contains one, or many strings (but not white space)    *     (             -- start of extension grouping    *        \.         -- existence of a dot, eg.: .kmz    *        (?i)       -- NOT case sensitive for the next group    *        (kmz|kml)  -- kmz OR kml strings    *     )$            -- should exist on the end    *  )                -- end of main grouping    */   public static String fileExtensionPattern(String[] extensions) {

      StringBuilder sb = new StringBuilder("");

      if (extensions.length > 0) {
         int n = 0;
         for (String extension : extensions) {
            if (n > 0) {
               sb.append("|"); // append OR            }
            sb.append(extension);
         }
      }
      String pattern = "([^\\s]+(\\.(?i)(" + sb.toString() + "))$)";
      System.out.println("fileExtensionPattern: " + pattern);
      return pattern;
   }

   public static final String anchorTagPattern = "<a *href=\"(.+?)</a>";
   public static final String urlPattern = "(https?|ftp|file)://[-a-zA-Z0-9+&@#/%?=~_|!:,.;]*[-a-zA-Z0-9+&@#/%=~_|]";
}

Java: RegEx: WebSpider

In this tutorial we will create a basis for a WebSpider program that fetches all links from the given Web page, checks which have extensions of interest and downloads these files.

The result of the app will be a folder with Google Earth map overlays:

/Users/uki/Desktop/KMZ

├── 1.kmz

├── 10.kmz

├── 11.kmz

├── 12.kmz

├── 13.kmz

├── 14.kmz

├── 15.kmz

├── 16.kmz

├── 17.kmz

├── 18.kmz

We will use basic java networking classes:

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.net.MalformedURLException;
import java.net.URL;

and regular expressions:

ublic class RegExConstants {

   /**    * @param extensions    * @return    *  (                -- start of main grouping    *     [^\s]+        -- must contains one, or many strings (but not white space)    *     (             -- start of extension grouping    *        \.         -- existence of a dot, eg.: .kmz    *        (?i)       -- NOT case sensitive for the next group    *        (kmz|kml)  -- kmz OR kml strings    *     )$            -- should exist on the end    *  )                -- end of main grouping    */   public static String fileExtensionPattern(String[] extensions) {

      StringBuilder sb = new StringBuilder("");

      if (extensions.length > 0) {
         int n = 0;
         for (String extension : extensions) {
            if (n > 0) {
               sb.append("|"); // append OR            }
            sb.append(extension);
         }
      }
      String pattern = "([^\\s]+(\\.(?i)(" + sb.toString() + "))$)";
      System.out.println("fileExtensionPattern: " + pattern);
      return pattern;
   }

   public static final String anchorTagPattern = "<a *href=\"(.+?)</a>";
   public static final String urlPattern = "(https?|ftp|file)://[-a-zA-Z0-9+&@#/%?=~_|!:,.;]*[-a-zA-Z0-9+&@#/%=~_|]";
}

Function To Censor String

Here is a function that censors a string, but leaves in all of the punctuation. It searches for strings of letters and numbers and if it matches "badword", will replace it with "[censored]".

public String censorString(String originalString)

{

StringBuffer orig = new StringBuffer(originalString);

Pattern p = Pattern.compile("[0-9/A-Z/a-z]+");

Matcher m = p.matcher(orig);

StringBuffer censor = new StringBuffer();

boolean result = m.find();

while (result)

{

String match = originalString.substring(m.start(), m.end());

if(match.equals("badword"))

m.appendReplacement(censor, "[censored]");

result = m.find();

}

m.appendTail(censor);

return censor.toString();

}

Function To Censor String

Here is a function that censors a string, but leaves in all of the punctuation. It searches for strings of letters and numbers and if it matches "badword", will replace it with "[censored]".

public String censorString(String originalString)

{

StringBuffer orig = new StringBuffer(originalString);

Pattern p = Pattern.compile("[0-9/A-Z/a-z]+");

Matcher m = p.matcher(orig);

StringBuffer censor = new StringBuffer();

boolean result = m.find();

while (result)

{

String match = originalString.substring(m.start(), m.end());

if(match.equals("badword"))

m.appendReplacement(censor, "[censored]");

result = m.find();

}

m.appendTail(censor);

return censor.toString();

}

Regular expression: user input validation checks

Validate email properties:

String regexExp = "^[a-zA-Z0-9]+[.a-zA-Z0-9_-]+@[a-zA-Z0_.-]+\\.[a-zA-Z]+$";

String regexExp2 = "^[a-zA-Z]+@[a-zA-Z0_.-]+\\.[a-zA-Z]+$";

String errMessage = "Invalid email address.";

Validate alpha numeric properties:

String regexExp = "[a-zA-z0-9]*";

Validate numeric properties:

String regexExp = "^[-+]?\\d*\\.?\\d*$";

Validate alpha properties:

String regexExp = "^([a-zA-Z\\s-\']+)$";

Validate URL properties:

String regexExp = "(HTTPS?://)[A-Z0-9.-]+\\.[A-Z]{2,6}([\\w\\d:#@%/;$()~_?\\+\\-=\\\\\\.&]*)";

String

Regular expression: user input validation checks

Validate email properties:

String regexExp = "^[a-zA-Z0-9]+[.a-zA-Z0-9_-]+@[a-zA-Z0_.-]+\\.[a-zA-Z]+$";

String regexExp2 = "^[a-zA-Z]+@[a-zA-Z0_.-]+\\.[a-zA-Z]+$";

String errMessage = "Invalid email address.";

Validate alpha numeric properties:

String regexExp = "[a-zA-z0-9]*";

Validate numeric properties:

String regexExp = "^[-+]?\\d*\\.?\\d*$";

Validate alpha properties:

String regexExp = "^([a-zA-Z\\s-\']+)$";

Validate URL properties:

String regexExp = "(HTTPS?://)[A-Z0-9.-]+\\.[A-Z]{2,6}([\\w\\d:#@%/;$()~_?\\+\\-=\\\\\\.&]*)";

String