• 搜狐网站诚聘Java、PHP和C++工程师
• 北京: 千橡集团暨校内网诚聘软件研发工程师

• 搜狐网站诚聘Java、PHP和C++工程师
• 北京: 千橡集团暨校内网诚聘软件研发工程师

• 搜狐网站诚聘Java、PHP和C++工程师
• 北京: 千橡集团暨校内网诚聘软件研发工程师

final ExecutorService Gathers = Executors.newFixedThreadPool(gatherNum);
while(true)
{
    Gathers.execute(new Gather());
}

• 北京: 千橡集团暨校内网诚聘软件研发工程师
• 搜狐网站诚聘Java、PHP和C++工程师

/**
	 * @functionName: generateMD5
	 * @description: generate the MD5 digest from 16 byte int to the ASCII format
	 * @version: 1.0
	 * @beCareful:16 byte int -> 32 byte ASCII 
	 * @param source
	 * @return  String the 32 byte ASCII MD5
	 */
	public static String generateMD5(String source)
	{
		byte[] bSource = source.getBytes();
		MessageDigest md = null;
		try
		{
			md = MessageDigest.getInstance("MD5");
		}
		catch (NoSuchAlgorithmException e)
		{
			e.printStackTrace();
		}
		byte[] md5 = md.digest(bSource);
		StringBuffer buff = new StringBuffer();
		for(int i = 0; i < md5.length; i++)
		{
			String byteStr = Integer.toHexString(md5[i] & 0xFF);
			if(byteStr.length() < 2) buff.append('0');
			buff.append(byteStr);
		}
		return buff.toString();
	}

private void fetch()
	{
		while(true)
		{
			try
			{
				new FetchWeb().fetch();
			}
			catch (URISyntaxException e)
			{
				Log4j.logger.debug(e);
				e.printStackTrace();
			}
		}
	}

public void fetch() throws URISyntaxException
	{
		String iteratorHashSetUvUrlsNext=UrlDB.pickUrl();
		System.out.println("the url we pick: " + iteratorHashSetUvUrlsNext);
		WebPageProcessFace tryit = new WebPageProcessFace(iteratorHashSetUvUrlsNext);
		try
		{
			if(tryit.doSomething())
			{
                                //问题会不会这？
				if(UrlDB.addUrl(iteratorHashSetUvUrlsNext, tryit.getLinks())) 
					Log4j.logger.info("addurl ok");
				tryit = null;
			}
			else
			{
				tryit = null;
			}
		}
		catch (IOException e)
		{
			tryit = null;
			e.printStackTrace();
			Log4j.logger.debug(e);
		}
		tryit = null;
		Log4j.logger.info("\n");
		Log4j.logger.info("\n");
	}

• 北京: 千橡集团暨校内网诚聘软件研发工程师
• 搜狐网站诚聘Java、PHP和C++工程师

private ArrayList<String> ParseHyperLinks() throws ParserException
	{
		ArrayList<String> hyperLinks = new ArrayList<String>();
		
		SimpleNodeIterator parseSimpleNodeIterator = null;
		if(getContentType().equalsIgnoreCase("text/html"))
		{
			
			try
			{
				Parser parse = new Parser(getHTMLEntity());
				parseSimpleNodeIterator = parse.parse(new TagNameFilter("A")).elements();
			}
			catch (ParserException e1)
			{
				Log4j.logger.debug(e1);
				e1.printStackTrace();
				throw new ParserException();
			}
			while(parseSimpleNodeIterator.hasMoreNodes())
			{
				String extractLink=new String();
				LinkTag link = (LinkTag) parseSimpleNodeIterator.nextNode();
				if(link.isHTTPLink())
				{
					 extractLink = link.extractLink().trim();
					// Log4j.logger.info(extractLink+"<<<<<<<<<<");
					if(extractLink.startsWith("#")) continue;
					if(extractLink.startsWith("*")) continue;
					if(extractLink.equals("/")) continue;
					if(isFilterLink(extractLink)) continue;
					if(extractLink.contains(" ")) extractLink = extractLink.split(" ")[0];
					if(extractLink.length() > 6)
					{
						if(!extractLink.substring(0, 4).equals("http"))
						{
							if(!extractLink.startsWith("/")) extractLink = "/" + extractLink;
							extractLink = url + extractLink;
						}
					}
					if(extractLink.length() <= 6)
					{
						if(!extractLink.startsWith("/")) extractLink = "/" + extractLink;
						extractLink = url + extractLink;
					}
					// Log4j.logger.info(">>>>>>>"+extractLink);
					hyperLinks.add(extractLink);
					extractLink=null;
				}
			}
		}
		return hyperLinks;
	}

public static boolean addUrl(String currentUrl, ArrayList<String> Links)
	{
		Log4j.logger.info("robotsMap: "+RobotsTxtCache.robotsMap.size());
		currentUrl=currentUrl.trim();
		// first put the just url into the v. collection
		vUrlsMD5Collection.put(MD5Generator.generateMD5(currentUrl), currentUrl);
		ArrayList<String> robotstxt;
		// process the hyberlinks
		for(int count=0;count<Links.size();count++)
		{
			String hl=new String();
				hl=Links.get(count);

			if(!vUrlsMD5Collection.containsKey(MD5Generator.generateMD5(hl)))
			{
				int weight = 10;
				Url u = null;
				try
				{
					u = new Url(hl);
				}
				catch (URISyntaxException e)
				{
					e.printStackTrace();
					Log4j.logger.debug(e);
					return false;
				}
			
				System.out.println(">>>>>"+hl);
					weight = u.getPageWeigth(new BFSStrategy());
					uvUrlsCollection.get(weight).add(hl);

			}

		}
		Links.clear();
		Links.trimToSize();
		Links=null;
		return true;
	}

• 北京: 千橡集团暨校内网诚聘软件研发工程师
• 搜狐网站诚聘Java、PHP和C++工程师

public class text
{
public static void main(String[] args) throws IOException
{
String hostname = "www.javaeye.com";
int port = 80;
InetAddress addr = InetAddress.getByName(hostname);
System.out.println(addr);
Socket s = new Socket(hostname, port);
// s.setSoTimeout(99999);
s.setKeepAlive(true);
if(s.getKeepAlive())
{
System.out.println("******************keep-alive*****************");
}
BufferedWriter wr = new BufferedWriter(new OutputStreamWriter(s.getOutputStream()));
wr.write("GET / HTTP/1.1" + "\r\n" + "Host: www.javaeye.com" + "\r\n" + "Connection: Keep-Alive" + "User-Agent: tse" + "\r\n");
wr.write("\r\n");
wr.flush();
BufferedReader br = new BufferedReader(new InputStreamReader(s.getInputStream()));
String line;
while((line = br.readLine()) != null)
{
System.out.println(line);
}
System.out.println(s.isClosed());

//以下想用复用同一个HOST＋port的socket,可是不行
wr.write("GET /search HTTP/1.0" + "\r\n" + "Host: www.javaeye.com" + "\r\n" + "Connection: Keep-Alive" + "User-Agent: tse" + "\r\n");
// wr.write("GET /");
// wr.write("HOST:" + hostname + "\r\n");
wr.write("\r\n");
wr.flush();
BufferedReader b = new BufferedReader(new InputStreamReader(s.getInputStream()));
String linee;
if(s.getInputStream().available()  > 0)
{
System.out.println("hava something");
}
while((linee = b.readLine()) != null)
{
System.out.println(linee);
}
}
}

• 搜狐网站诚聘Java、PHP和C++工程师
• 北京: 千橡集团暨校内网诚聘软件研发工程师