waldenlake

private ArrayList<String> ParseHyperLinks() throws ParserException
	{
		ArrayList<String> hyperLinks = new ArrayList<String>();
		
		SimpleNodeIterator parseSimpleNodeIterator = null;
		if(getContentType().equalsIgnoreCase("text/html"))
		{
			
			try
			{
				Parser parse = new Parser(getHTMLEntity());
				parseSimpleNodeIterator = parse.parse(new TagNameFilter("A")).elements();
			}
			catch (ParserException e1)
			{
				Log4j.logger.debug(e1);
				e1.printStackTrace();
				throw new ParserException();
			}
			while(parseSimpleNodeIterator.hasMoreNodes())
			{
				String extractLink=new String();
				LinkTag link = (LinkTag) parseSimpleNodeIterator.nextNode();
				if(link.isHTTPLink())
				{
					 extractLink = link.extractLink().trim();
					// Log4j.logger.info(extractLink+"<<<<<<<<<<");
					if(extractLink.startsWith("#")) continue;
					if(extractLink.startsWith("*")) continue;
					if(extractLink.equals("/")) continue;
					if(isFilterLink(extractLink)) continue;
					if(extractLink.contains(" ")) extractLink = extractLink.split(" ")[0];
					if(extractLink.length() > 6)
					{
						if(!extractLink.substring(0, 4).equals("http"))
						{
							if(!extractLink.startsWith("/")) extractLink = "/" + extractLink;
							extractLink = url + extractLink;
						}
					}
					if(extractLink.length() <= 6)
					{
						if(!extractLink.startsWith("/")) extractLink = "/" + extractLink;
						extractLink = url + extractLink;
					}
					// Log4j.logger.info(">>>>>>>"+extractLink);
					hyperLinks.add(extractLink);
					extractLink=null;
				}
			}
		}
		return hyperLinks;
	}

public static boolean addUrl(String currentUrl, ArrayList<String> Links)
	{
		Log4j.logger.info("robotsMap: "+RobotsTxtCache.robotsMap.size());
		currentUrl=currentUrl.trim();
		// first put the just url into the v. collection
		vUrlsMD5Collection.put(MD5Generator.generateMD5(currentUrl), currentUrl);
		ArrayList<String> robotstxt;
		// process the hyberlinks
		for(int count=0;count<Links.size();count++)
		{
			String hl=new String();
				hl=Links.get(count);

			if(!vUrlsMD5Collection.containsKey(MD5Generator.generateMD5(hl)))
			{
				int weight = 10;
				Url u = null;
				try
				{
					u = new Url(hl);
				}
				catch (URISyntaxException e)
				{
					e.printStackTrace();
					Log4j.logger.debug(e);
					return false;
				}
			
				System.out.println(">>>>>"+hl);
					weight = u.getPageWeigth(new BFSStrategy());
					uvUrlsCollection.get(weight).add(hl);

			}

		}
		Links.clear();
		Links.trimToSize();
		Links=null;
		return true;
	}