import java.io.*;
import java.util.*;
import java.net.*;
//_____________________________________________________________
public class words
{ 
//_____________________________________________________________
public static final void main(String[] args) throws Exception
{
   words bnk = new words();
   bnk.start(args);
}
//_____________________________________________________________
words() throws Exception 
{
}
//___________________________________________________________________________
void start(String[] args)
{
   StringBuffer buf = new StringBuffer();
   File fileb = new File("words.txt");
   if (!fileb.exists())
   {
      File file = new File("full.lst");
      if (!file.exists())
      {
         buf = new StringBuffer();
         String name = "http://www.textfiles.com/etext/FICTION/";
         if (args.length > 0) name = args[0];
         String file2 = getUrl(name);
         buf.append(file2);
         String[] list = links(file2,name).split("\r\n");
         for (int i = 0; i < list.length; i++)
         {
            System.out.println(list[i]);
            file2 = getUrl(list[i]);
            buf.append(file2);
         }
         saveFile("full.lst",buf.toString(),false);
      }
      String str = getFile("full.lst");
      String[] words = getwords(str).split(" ");
      HashMap hash = new HashMap();
      int hashpos = 0;
      buf = new StringBuffer();
      for (int i = 0; i < words.length; i++)
      {
         if (hash.get(words[i]) == null)
         {
            hash.put(words[i],hashpos++);
            if (words[i].trim().length() >= 3)
            {
               buf.append(words[i].trim() + "\r\n");
            }
         }
      }
      String[] list = buf.toString().split("\r\n");
      Arrays.sort(list);
      buf = new StringBuffer();
      for (int i = 0; i < list.length; i++)
      {
         buf.append(list[i] + "\r\n");
      }
      saveFile("words.txt",buf.toString(),false);
   }
}
//___________________________________________________________________________
String links(String file, String name)
{
   file = file.toLowerCase();
   StringBuffer buf = new StringBuffer();
   int poslast = name.lastIndexOf("/");
   String parent = name.substring(0,poslast+1);
   if (parent.length() <= 8) parent = name + "/";
   int pos = 0;
   for (int i = 0; i < 1000; i++)
   {
      pos = file.indexOf("<a",pos+1);
      if (pos < 0) break;
      int pos2 = file.indexOf("href",pos+1);
      int pos3 = file.indexOf("\"",pos+1);
      int pos4 = file.indexOf("'",pos+1);
      int pos5 = 0;
      int pos6 = 0;
      if ((pos2 > pos) & (pos2 < pos + 20))
      {
         String lk = "\"";
         if ((pos3 > pos2+1) & (pos3 < pos4))
         {
            pos6 = pos3;
            pos5 = file.indexOf("\"",pos3 + 1);
         }
         else
         {
             if (pos4 > pos2+1)
             {
                pos6 = pos4;
                pos5 = file.indexOf("'",pos4 + 1);
             }
         }
         if (pos5 > pos)
         {
            String htm = file.substring(pos6+1,pos5);
            if (htm.indexOf("\r") < 0)
            {
               if (htm.indexOf("\n") < 0)
               {
                  if (htm.length() > 4)
                  {
                     if (!htm.substring(0,4).equals("http"))
                     {
                        htm = parent + htm;
                     }
                  }
                  else htm = parent + htm;
                  buf.append(htm + "\r\n");
               }
            }
         }
      }
   }
   return(buf.toString());
}
//___________________________________________________________________________
String getUrl(String name)
{
   StringBuffer buf = new StringBuffer();
  try
  {
   String line;
   URL url = new URL(name);
   InputStream is = url.openStream();
   BufferedReader br = new BufferedReader(new InputStreamReader(is));
   while ((line = br.readLine()) != null) 
   {
      buf.append(line + "\r\n");
   }
  }
  catch (Exception e)
  {
   System.out.println(e);
  }
   return(buf.toString());
}
//___________________________________________________________________________
String getwords(String str)
{
   byte[] bb = str.toLowerCase().getBytes();
   StringBuffer buf = new StringBuffer();
   for (int i = 0; i < bb.length; i++)
   {
      if (((bb[i] >= 'a') & (bb[i] <= 'z')) | (bb[i] == ' '))
      {
         buf.append((char)bb[i]);
      } 
      else buf.append(" ");
   }
   String out = buf.toString();
   for (int i = 0; i < 10000; i++)
   {
      if (out.indexOf("  ") < 0) break;
      out = out.replace("  "," ");
   }
   return(out);
}
//___________________________________________________________________________
String getFile(String file) 
{
   StringBuffer buf=new StringBuffer();String str;
  try
  {
   BufferedReader in = new BufferedReader (new FileReader (file));
   while((str=in.readLine())!=null)
   {
      buf.append(str+"\r\n");
   }
   in.close();
  }
  catch (Exception e)
  {
   System.out.println(e);
  }
   return (buf.toString());
}
//___________________________________________________________________________
void saveFile(String file, String str, boolean append) 
{
  try
  {
   BufferedWriter out = new BufferedWriter(new FileWriter(file, append));
   out.write(str);
   out.close();
  }
  catch (Exception e)
  {
   System.out.println(e);
  }
}
}//__________________________________________________________________________