package market.thread;

import java.net.*;
import market.util.*;
import market.*;
import java.io.*;
import market.util.text.*;

/**
 * Download web page from Internet or get text and link from m_docs,
 * if web page is not modified after last downloaded this site.
 * Creation date: (1/2/00 9:10:38 PM)
 * @author: tin
 */
public class Downloader extends Thread {

	public LangTable langTable; // This is a table whit Language
	
	public Queue  srcInStream; // In this stream Netrunner write source to grab


	public Queue htmlOutStream;	
//	public PipeSinhro srcStreamSinhro;
	public boolean working;
	public boolean readinFromStream;

	public int startPosition = ConfigMarket.startPosForCheck;
	public int stopPosition =  ConfigMarket.stopPosForCheck;
	protected SrcElement baseSrc = null;
/**
 * Downloader constructor comment.
 */
public Downloader() {
	super();
}
/**
 * This constructor get input pipe. By this pipe Downloader recive source to grab. 
 * Creation date: (12/27/99 6:42:25 PM)
 * @param num int ( Netrunner number )
 * @param threadGroup ThreadGroup 
 * @param srcInPipe market.util.Queue
 * @param langTable LangTable
 * @param baseSrc SrcElement
 * @exception java.io.IOException The exception description.
 * @exception java.io.StreamCorruptedException The exception description.
 */
public Downloader( int num, ThreadGroup threadGroup, Queue srcInPipe, LangTable langTable, SrcElement baseSrc ) throws IOException, StreamCorruptedException {
	super( threadGroup, "Downloader"+num);
	this.langTable = langTable;
	this.srcInStream= srcInPipe;
	this.baseSrc = baseSrc;
	readinFromStream = false;
	
// Html chenal set 	

	htmlOutStream = new Queue( ConfigMarket.DOWNLOADER_OUT );
// end Html chenal set 	

}
/**
 * Downloader constructor comment.
 * @param target java.lang.Runnable
 */
public Downloader(Runnable target) {
	super(target);
}
/**
 * Get date of docoment in market_DB 
 * Gives a time as the number of seconds since January 1, 1970, GMT.
 * Creation date: (1/6/00 6:02:12 PM)
 * @return long
 * @param srcKey int
 */
public long getDataSrc(int srcKey) {
	return 0;
}
/**
 * Return html docoment stream.
 * Creation date: (1/2/00 10:47:50 PM)
 * @return market.util.Queue
 */
public Queue getHtmlInPipe() {
	return htmlOutStream;
}
/**
 * Return TRUE if contentType is acceptable.
 * Creation date: (27.1.2000 “. 18:04:00)
 * @return boolean
 * @param contentType java.lang.String
 */
public boolean IsContentTypeOk(String contentType) {
	
	for (int i = 0; i < ConfigMarket.contentType.length; i++) {
		if (contentType.equalsIgnoreCase(ConfigMarket.contentType[i])) {
			return true;
		}
	}
	return false;
}
/**
 * Insert the method's description here.
 * Creation date: (13.1.00 18:45:58)
 * @return boolean
 */
public boolean isWait() {
	return readinFromStream;
}
/**
 * Body on Downloader  <br>
 * 1. Get next source to download <br>
 * 2. Connect <br>
 * 3. Download page  <br>
 * 4. Send to Parser  <br>
 * Creation date: (1/3/00 8:18:44 PM)
 */
public void run() {
	boolean fromDB, firstDownload;
	String resStatus;
	String contentType;
	String html;
	byte[] bytes = new byte[Const.SIZE_FILE_BUFFER];
	int length, allLength;
	LinkElement inSrc = null;
	working = true;
	Runtime myRuntime = Runtime.getRuntime();
	myRuntime.traceInstructions(true);
	myRuntime.traceMethodCalls(true);
	try {
		while (working) {
			if (myRuntime.freeMemory() / myRuntime.totalMemory() < Const.MinFreeMemory) {
				ConfigMarket.logWriter.log(getName() + ": Total Memory: " + myRuntime.totalMemory() + " Free Memory: " + myRuntime.freeMemory(), LogPrinter.DEBUG);
				System.runFinalization();
				System.gc();
			}

			// * 1. Get next source to download 
			try {
				inSrc = null;
				ConfigMarket.logWriter.log(getName() + ": Geting Next Link...", LogPrinter.DEBUG);
				readinFromStream = true;
				inSrc = (LinkElement) srcInStream.readObject();
				readinFromStream = false;
				if (inSrc == null) {
					working = false;
					ConfigMarket.logWriter.log(getName() + ": Finish link=" + srcInStream.getNumOfElement(), LogPrinter.DEBUG);
					continue;
				}
				ConfigMarket.logWriter.log(getName() + ": Next link is srcValue=" + inSrc.srcString, LogPrinter.INFO);
			} catch (Exception e) {
				ConfigMarket.logWriter.log(getName() + ": Empty link Stream!", LogPrinter.ERROR);
				working = false;
				continue;
			}


			// * 2. Connect
			URL url_SRC;
			HttpURLConnection HTMLFileconn;
			DataInputStream HTMLFileStream = null;
			M_Docs_Result res;
			try {
				url_SRC = new URL(inSrc.srcString);
				ConfigMarket.logWriter.log(getName() + ": Try OpenConnection ...", LogPrinter.DEBUG);
				HTMLFileconn = (HttpURLConnection) url_SRC.openConnection();
				setHttpHeader(HTMLFileconn, inSrc); // Set HTTP header
				ConfigMarket.logWriter.log(getName() + ": Try connect ...", LogPrinter.DEBUG);
				HTMLFileconn.connect(); // Connect to page

				int resCode = HTMLFileconn.getResponseCode();
				switch (resCode) {
					case (HTMLFileconn.HTTP_OK) :
						{
							break;
						}
					case (HTMLFileconn.HTTP_NOT_MODIFIED) :
						{
							break;							
						}
					default:{
						resStatus = HTMLFileconn.getHeaderField(0);
						ConfigMarket.logWriter.log(getName() + ": Can't get URL: " + inSrc.srcString + " HTTP Result:" + resStatus, LogPrinter.ERROR);
						//Send error code (HTTP status code)
						ConfigMarket.mDocsCacher.tryFail(inSrc.srcKey, resStatus);
						continue;
					}	
				}
				
				contentType = HTMLFileconn.getContentType(); 
				resStatus = " Unacceptable content type : " + contentType ;	
				if (!IsContentTypeOk(contentType)) {
					ConfigMarket.logWriter.log(getName() + ": Unacceptable content type on : " + inSrc.srcString + " on domain.", LogPrinter.ERROR);
					//Store error in m_docs
					ConfigMarket.mDocsCacher.tryFail(inSrc.srcKey, resStatus);
					continue;
				}
								
				long time = HTMLFileconn.getLastModified();
				ConfigMarket.logWriter.log(getName() + ": Try get from Cache ...", LogPrinter.DEBUG);
				//Try get from Cache ...
				res = ConfigMarket.mDocsCacher.getDocument(inSrc.srcKey, time);
				if (Const.DOWNLOAD_FAILED == res.status) {
					//!!! if docoment can't download set info in log file	   			
					ConfigMarket.logWriter.log(getName() + ": On srcKey=" + inSrc.srcKey + " have download error.", LogPrinter.DEBUG);
					continue;
				}
				if (Const.LOAD_FAILED == res.status) {
					//!!! if docoment can't download set info in log file	   			
					ConfigMarket.logWriter.log(getName() + ": On srcKey=" + inSrc.srcKey + " have load error.", LogPrinter.DEBUG);
				}
				html = res.document;
				if (html == null) {
					HTMLFileStream = new DataInputStream(HTMLFileconn.getInputStream());
					fromDB = false;
				} else {
					fromDB = true;
				}
			} catch (IOException e) {
				ConfigMarket.logWriter.log(getName() + ": Can't connect to: " + inSrc.srcString, LogPrinter.ERROR);
				ConfigMarket.mDocsCacher.tryFail(inSrc.srcKey, e.toString());
				//Maybe send error code (HTTP status code)
				continue;
			}
			//			catch (Throwable t) {
			//				ConfigMarket.logWriter.log(getName() + ": Has error: " + t, LogPrinter.ERROR);
			//				t.printStackTrace(ConfigMarket.logWriter.getPrintWriter());
			//				ConfigMarket.mDocsCacher.tryFail(inSrc.srcKey);
			//				//Maybe send error code (HTTP status code)
			//				working = false;
			//				return; // exit
			//			}

			// Check data and then download

			// * 3. Download page

			if (!fromDB && (HTMLFileStream != null)) {
				html = "";
				allLength = 0;
				firstDownload = true;
				ConfigMarket.logWriter.log(getName() + ": Try get html page ...", LogPrinter.DEBUG);
				try {
					while ((length = HTMLFileStream.read(bytes)) != -1) {
						// Check size
						allLength += length;
						if (allLength >= ConfigMarket.MAX_DOC_SIZE) {
							break;
						}
						html = html + new String(bytes, 0, length);
						// Check for binary data
						if (firstDownload) {
							firstDownload = false;
							// ??? Check sentances for binary data. what I do? 
							Text.sentanceCheck(html, ConfigMarket.AcceptCharEnum, startPosition, stopPosition);
						}
						yield();
					}
					HTMLFileStream.close();
					HTMLFileStream = null;
				} catch (IOException e) {
					ConfigMarket.logWriter.log(getName() + ": Can't read from web page: " + inSrc.srcString, LogPrinter.ERROR);
					ConfigMarket.mDocsCacher.tryFail(inSrc.srcKey, e.toString());
					continue;
				}
			}
			// * 4. Send to Parser

			HtmlElement htmlElem = new HtmlElement(html, inSrc.srcString, fromDB, inSrc.srcKey, inSrc.grabLevel, inSrc.langKey, inSrc.parSrcKey);
			try {
				ConfigMarket.logWriter.log(getName() + ": Sending element... :", LogPrinter.DEBUG);
				htmlOutStream.writeObject((Object) htmlElem);
				//				countSource( inSrc.srcKey, );
				int numOutElem = htmlOutStream.getNumOfElement();
				ConfigMarket.logWriter.log(getName() + ": numOf Out Element:" + numOutElem, LogPrinter.DEBUG);
				if (numOutElem > Const.MAX_DOWNLOAD_ELEM)
					sleep(Const.DOWNLOADER_TIME_SLEEP);
				else
					continue;
			} catch (InterruptedException e) {
				continue;
			} catch (Exception e) {
				ConfigMarket.logWriter.log(getName() + ": Can't write in html stream: " + inSrc.srcString, LogPrinter.ERROR);
				e.printStackTrace(ConfigMarket.logWriter.getPrintWriter());
				continue;
			}
		}
		htmlOutStream.writeObject((Object) null); // Send mesage to Parser for End
	} catch (ThreadDeath t) {
		ConfigMarket.logWriter.log(getName() + ": Force stop (kill):", LogPrinter.ERROR);
		t.printStackTrace(ConfigMarket.logWriter.getPrintWriter());
		ConfigMarket.logWriter.log(getName() + ": Finish. ", LogPrinter.INFO);
		return;
	} catch (Throwable t) {
		ConfigMarket.logWriter.log(getName() + ": Somting wrong : " + t, LogPrinter.ERROR);
		t.printStackTrace(ConfigMarket.logWriter.getPrintWriter());
		( (Graber)getThreadGroup() ).sendError();
	}
	ConfigMarket.logWriter.log(getName() + ": Finish.", LogPrinter.INFO);
}
/**
 * Set http header.
 * Creation date: (08.7.2000 “. 13:51:02)
 * @return boolean
 * @param conn java.net.URLConnection
 * @param inSrc market.util.LinkElement
 */
public boolean setHttpHeader(URLConnection conn, LinkElement inSrc ) {
	conn.setRequestProperty("user-agent", baseSrc.userAgent );
	conn.setRequestProperty("Accept", "text/html, text/plain");
	conn.setRequestProperty("Accept-Language", "en");//langTable.ltResolveShortNameToKey( inSrc.langKey ));
	//conn.setIfModifiedSince( getDataSrc( inSrc.srcKey ) );

	return true;
}
}