#define LOCAL_DEBUG
#include "debug.h"

#include "config.h"
#include "acfg.h"
#include "dlcon.h"

#include "fileitem.h"
#include "fileio.h"
#include "sockio.h"

using namespace MYSTD;

// evil hack to simulate random disconnects
//#define DISCO_FAILURE

static const cmstring sGenericError("567 Unknown download error occured");

dlcon::dlcon(bool bManualExecution, string *xff) :
		m_bStopASAP(false), m_bManualMode(bManualExecution)
{
	LOGSTART("dlcon::dlcon");
	m_wakepipe[0] = m_wakepipe[1] = -1;
	if (0 == pipe(m_wakepipe))
	{
		set_nb(m_wakepipe[0]);
		set_nb(m_wakepipe[1]);
	}
	if (xff)
		m_sXForwardedFor = *xff;
}

struct tDlJob
{
	tFileItemPtr m_pStorage;
	mstring sErrorMsg;
	inline bool HasBrokenStorage()
	{
		return (!m_pStorage || m_pStorage->GetStatus() >= FIST_LOCALERROR);
	}

#define EFLAG_MORE 0
#define EFLAG_DONE 1
#define EFLAG_DO_DISCON 2
#define EFLAG_JOB_BROKEN 4
#define EFLAG_MIRROR_BROKEN 8
#define EFLAG_STORE_COLLISION 16
#define EFLAG_HINT_SWITCH 32
#define EFLAG_LOST_CON 64
#define EFLAG_KILL_LAST_FILE 128

	inline const mstring GetPeerName()
	{
		return m_pCurBackend ? m_pCurBackend->sHost : m_fileUri.sHost;
	}

	inline const MYSTD::string GetPeerPort()
	{
		cmstring &ps = m_pCurBackend ? m_pCurBackend->sPort : m_fileUri.sPort;
		return ps.empty() ? acfg::remoteport : ps;
	}

	inline acfg::tRepoData::IHookHandler * GetConnStateTracker()
	{
		return m_pBEdata ? m_pBEdata->m_pHooks : NULL;
	}

	typedef enum
	{
		STATE_GETHEADER, STATE_REGETHEADER, STATE_PROCESS_DATA,
		STATE_GETCHUNKHEAD, STATE_PROCESS_CHUNKDATA, STATE_GET_CHUNKTRAILER,
		STATE_FINISHJOB
	} tDlState;

	const acfg::tRepoData * m_pBEdata;

	tHttpUrl m_fileUri;
	const tHttpUrl *m_pCurBackend;

	uint_fast8_t m_eReconnectASAP;

	off_t m_nRest;

	tDlState m_DlState;

	void Init()
	{
		m_nRest = 0;
		m_eReconnectASAP = 0;
		m_DlState = STATE_GETHEADER;
		m_pCurBackend = NULL;
		if (m_pStorage)
			m_pStorage->IncDlRefCount();
	}

	tDlJob(dlcon *p, tFileItemPtr pFi, const tHttpUrl& uri) :
			m_pStorage(pFi), m_pBEdata(NULL),
			m_fileUri(uri)
	{
		Init();
	}

	tDlJob(dlcon *p, tFileItemPtr pFi, const acfg::tRepoData * pBackends,
			const MYSTD::string & sPath) :
			m_pStorage(pFi), m_pBEdata(pBackends), m_fileUri("", sPath)
	{
		Init();
	}

	~tDlJob()
	{
		if (m_pStorage && FIST_COMPLETE != m_pStorage->GetStatus())
			m_pStorage->DecDlRefCount(sErrorMsg.empty() ? sGenericError : sErrorMsg);
	}
//
//acfg::tRepoData::IHookHandler * tDlJob::GetConnStateObserver()
//{
//	return m_pBEdata ? m_pBEdata->m_pHooks : NULL;
//}

//
//void tDlJob::BlacklistBackend()
//{
//	m_parent->m_MirrorHostBlacklist.insert(m_pCurBackend
//			? m_pCurBackend->sHost
//		    : m_fileUri.sHost);
//}

// needs connectedHost, blacklist, output buffer from the parent, proxy mode?
	void AppendRequest(tSS &head, cmstring &xff)
	{
		LOGSTART("tDlJob::AppendRequest");

		bool proxied = ! acfg::proxy_info.sHost.empty();

		head << (m_pStorage->m_bHeadOnly ? "HEAD " : "GET ");
		if (!proxied) // use absolute path
		{
			if (m_pCurBackend) // base dir from backend definition
				head << m_pCurBackend->sPath;
			head << m_fileUri.sPath;
		}
		else
			head << RemoteUri();

		head << " HTTP/1.1\r\n" << acfg::agentheader << "Host: " << GetPeerName() << "\r\n";

		if (proxied) // proxy stuff, and add authorization if there is any
		{
			ldbg("using proxy");
			head << acfg::proxy_info.sPath << "Proxy-Connection: keep-alive\r\n";
		}

		// either by backend or by host in file uri, never both
		ASSERT( (m_pCurBackend && m_fileUri.sHost.empty()) || (!m_pCurBackend && !m_fileUri.sHost.empty()));

		if (m_pStorage->m_nSizeSeen > 0 || m_pStorage->m_nRangeLimit >=0)
		{
			bool bSetRange(false), bSetIfRange(false);

			lockguard g(m_pStorage.get());
			const header &pHead = m_pStorage->GetHeaderUnlocked();

			if (m_pStorage->m_bCheckFreshness)
			{
				if (pHead.h[header::LAST_MODIFIED])
				{
					if (acfg::vrangeops > 0)
					{
						bSetIfRange = true;
						bSetRange = true;
					}
					else if(acfg::vrangeops == 0)
					{
						head << "If-Modified-Since: " << pHead.h[header::LAST_MODIFIED] << "\r\n";
					}
				}
			}
			else
			{
				/////////////// this was protection against broken stuff in the pool ////
				// static file type, date does not matter. check known content length, not risking "range not satisfiable" result
				//
				//off_t nContLen=atol(h.get("Content-Length"));
				//if (nContLen>0 && j->m_pStorage->m_nFileSize < nContLen)
				bSetRange = true;
			}

			/*
			if(m_pStorage->m_nSizeSeen >0 && m_pStorage->m_nRangeLimit>=0)
			{
				bool bSaneRange=m_pStorage->m_nRangeLimit >=m_pStorage->m_nSizeSeen;
				// just to be sure
				ASSERT(bSaneRange);
			}
			if(m_pStorage->m_nRangeLimit < m_pStorage->m_nSizeSeen)
				bSetRange = bSetIfRange = false;
*/

			/* use APT's old trick - set the starting position one byte lower -
			 * this way the server has to send at least one byte if the assumed
			 * position is correct, and we never get a 416 error (one byte
			 * waste is acceptable).
			 * */
			if (bSetRange)
			{
				head << "Range: bytes=" << std::max(off_t(0), m_pStorage->m_nSizeSeen - 1) << "-";
				if(m_pStorage->m_nRangeLimit>=0)
					head << m_pStorage->m_nRangeLimit;
				head << "\r\n";
			}

			if (bSetIfRange)
				head << "If-Range: " << pHead.h[header::LAST_MODIFIED] << "\r\n";
		}

		if (m_pStorage->m_bCheckFreshness)
			head << "Cache-Control: no-store,no-cache,max-age=0\r\n";

		if (acfg::exporigin && !xff.empty())
			head << "X-Forwarded-For: " << xff << "\r\n";

		head << acfg::requestapx << "Accept: */*\r\nConnection: Keep-Alive\r\n\r\n";

		ldbg("Request cooked, buffer contents: " << head);
#ifdef SPAM
		//head.syswrite(2);
#endif

	}

	inline string RemoteUri()
	{
		return (m_pCurBackend ? m_pCurBackend->ToURI() + m_fileUri.sPath : m_fileUri.ToURI());
	}

	uint_fast8_t NewDataHandler(acbuf & inBuf)
	{
		LOGSTART("tDlJob::NewDataHandler");
		while (true)
		{
			off_t nToStore = min((off_t) inBuf.size(), m_nRest);
			ldbg("To store: " <<nToStore);
			if (0 == nToStore)
				break;

			if (!m_pStorage->StoreFileData(inBuf.rptr(), nToStore))
				return EFLAG_DO_DISCON | EFLAG_JOB_BROKEN;

			m_nRest -= nToStore;
			inBuf.drop(nToStore);
		}

		ldbg("Rest: " << m_nRest);

		if (m_nRest == 0)
		{
			m_DlState = (STATE_PROCESS_DATA == m_DlState) ? STATE_FINISHJOB : STATE_GETCHUNKHEAD;
		}
		else
			return EFLAG_MORE; // will come back

		return EFLAG_HINT_SWITCH;
	}

	/*!
	 *
	 * Process new incoming data and write it down to disk or other receivers.
	 */
	uint_fast8_t ProcessIncomming(acbuf & inBuf)
	{
		LOGSTART("tDlJob::ProcessIncomming");
		if (!m_pStorage)
			return EFLAG_DO_DISCON | EFLAG_JOB_BROKEN;

		for (;;) // returned by explicit error (or get-more) return
		{
			ldbg("switch: " << m_DlState);

			if (STATE_GETHEADER == m_DlState ||  STATE_REGETHEADER == m_DlState)
			{
				ldbg("STATE_GETHEADER");
				header h;
				if (inBuf.size() == 0)
					return EFLAG_MORE;

				bool bHotItem = (m_DlState == STATE_REGETHEADER);
				dbgline;

				int l = h.LoadFromBuf(inBuf.rptr(), inBuf.size());
				if (0 == l)
					return EFLAG_MORE;
				if (l<0)
				{
					dbgline;
					sErrorMsg = "500 Invalid header";
					// can be followed by any junk... drop that mirror, previous file could also contain bad data
					return EFLAG_MIRROR_BROKEN | EFLAG_DO_DISCON | EFLAG_KILL_LAST_FILE;
				}

				ldbg("contents: " << MYSTD::string(inBuf.rptr(), l));
				inBuf.drop(l);
				if (h.type != header::ANSWER)
				{
					sErrorMsg = "500 Unexpected response type";
					// smells fatal...
					return EFLAG_MIRROR_BROKEN | EFLAG_DO_DISCON;
				}
				ldbg("GOT, parsed: " << h.frontLine);

				// explicitly blacklist mirror if key file is missing
				if (h.getStatus() >= 400 && m_pBEdata)
				{
					for (tStrVecIterConst it = m_pBEdata->m_keyfiles.begin();
							it != m_pBEdata->m_keyfiles.end(); ++it)
					{
						if (endsWith(m_fileUri.sPath, *it))
						{
							sErrorMsg = "500 Bad mirror";
							return EFLAG_DO_DISCON | EFLAG_MIRROR_BROKEN;
						}
					}
				}

				const char *pCon = h.h[header::CONNECTION];
				if(!pCon)
					pCon = h.h[header::PROXY_CONNECTION];
				if (pCon && 0 == strcasecmp(pCon, "close"))
				{
					ldbg("Peer wants to close connection after request");
					m_eReconnectASAP = EFLAG_DO_DISCON;
				}

				if (m_pStorage->m_bHeadOnly)
				{
					m_DlState = STATE_FINISHJOB;
				}
				// the only case where we expect a 304
				else if(h.getStatus() == 304 && acfg::vrangeops == 0)
				{
					m_pStorage->SetupComplete();
					m_DlState = STATE_FINISHJOB;
				}
				else if (h.h[header::TRANSFER_ENCODING]
						&& 0 == strcasecmp(h.h[header::TRANSFER_ENCODING], "chunked"))
				{
					m_DlState = STATE_GETCHUNKHEAD;
					h.del(header::TRANSFER_ENCODING); // don't care anymore
				}
				else
				{
					dbgline;
					if (!h.h[header::CONTENT_LENGTH])
					{
						sErrorMsg = "500 Missing Content-Length";
						return EFLAG_DO_DISCON | EFLAG_JOB_BROKEN;
					}
					// may support such endless stuff in the future but that's too unreliable for now
					m_nRest = atoofft(h.h[header::CONTENT_LENGTH]);
					m_DlState = STATE_PROCESS_DATA;
				}
				h.set(header::XORIG, RemoteUri());
				if ( ! m_pStorage->DownloadStartedStoreHeader(h, inBuf.rptr(), bHotItem ))
				{
					ldbg("Item dl'ed by others or in error state --> drop it, reconnect");
					m_DlState = STATE_PROCESS_DATA;
					return EFLAG_DO_DISCON | EFLAG_JOB_BROKEN | EFLAG_STORE_COLLISION;
				}
			}
			else if (m_DlState == STATE_PROCESS_CHUNKDATA || m_DlState ==  STATE_PROCESS_DATA)
			{
				// similar states, just handled differently afterwards
				ldbg("STATE_GETDATA");
				uint_fast8_t res = NewDataHandler(inBuf);
				if (EFLAG_HINT_SWITCH != res)
					return res;
			}
			else if (m_DlState == STATE_FINISHJOB)
			{
				ldbg("STATE_FINISHJOB");
				m_DlState = STATE_GETHEADER;
				m_pStorage->StoreFileData(NULL, 0);
				return EFLAG_DONE | m_eReconnectASAP;
			}
			else if (m_DlState == STATE_GETCHUNKHEAD)
			{
				ldbg("STATE_GETCHUNKHEAD");
				// came back from reading, drop remaining newlines?
				while (inBuf.size() > 0)
				{
					char c = *(inBuf.rptr());
					if (c != '\r' && c != '\n')
						break;
					inBuf.drop(1);
				}
				const char *crlf(0), *pStart(inBuf.c_str());
				if (!inBuf.size() || NULL == (crlf = strstr(pStart, "\r\n")))
				{
					inBuf.move();
					return EFLAG_MORE;
				}
				UINT len(0);
				if (1 != sscanf(pStart, "%x", &len))
					return EFLAG_JOB_BROKEN; // hm...?
				inBuf.drop(crlf + 2 - pStart);
				if (len > 0)
				{
					m_nRest = len;
					m_DlState = STATE_PROCESS_CHUNKDATA;
				}
				else
					m_DlState = STATE_GET_CHUNKTRAILER;
			}
			else if (m_DlState == STATE_GET_CHUNKTRAILER)
			{
				if (inBuf.size() < 2)
					return EFLAG_MORE;
				const char *pStart(inBuf.c_str());
				const char *crlf(strstr(pStart, "\r\n"));
				if (!crlf)
					return EFLAG_MORE;

				if (crlf == pStart) // valid empty line -> done here
				{
					inBuf.drop(2);
					m_DlState = STATE_FINISHJOB;
				}
				else
					inBuf.drop(crlf + 2 - pStart); // drop line and watch for others
			}
		}
		ASSERT(!"unreachable");
		return EFLAG_JOB_BROKEN;
	}

private:
	// not to be copied ever
	tDlJob(const tDlJob&);
	tDlJob & operator=(const tDlJob&);
};


inline void dlcon::BlacklistMirror(tDlJobPtr & job, cmstring &msg)
{
	LOGSTART2("dlcon::BlacklistMirror", "what? [" <<
			job->GetPeerName() << "]:" <<
			job->GetPeerPort());
	m_blacklist[make_pair(job->GetPeerName(), job->GetPeerPort())] = msg;
}


inline bool dlcon::SetupJobConfig(tDlJobPtr &job, mstring *pReasonMsg)
{
	LOGSTART("dlcon::SetupJobConfig");
	// using backends? Find one which is not blacklisted

	MYMAP<MYSTD::pair<cmstring,cmstring>, mstring>::const_iterator bliter;

	if (job->m_pBEdata)
	{
		// keep the existing one if possible
		if (job->m_pCurBackend)
		{
			LOG("Checking [" << job->m_pCurBackend->sHost << "]:" << job->m_pCurBackend->sPort);
			bliter = m_blacklist.find(make_pair(job->m_pCurBackend->sHost, job->m_pCurBackend->sPort));
			if(bliter == m_blacklist.end())
				return true;
		}

		for (vector<tHttpUrl>::const_iterator it=job->m_pBEdata->m_backends.begin();
				it!=job->m_pBEdata->m_backends.end(); ++it)
		{
			bliter = m_blacklist.find(make_pair(it->sHost, it->sPort));
			if(bliter == m_blacklist.end())
			{
				job->m_pCurBackend = &(*it);
				return true;
			}

			if(pReasonMsg)
				*pReasonMsg = bliter->second;
		}
		return false;
	}

	// ok, look for the mirror data itself
	bliter = m_blacklist.find(make_pair(job->GetPeerName(), job->GetPeerPort()));
	if(bliter == m_blacklist.end())
		return true;
	else
	{
		if(pReasonMsg)
			*pReasonMsg = bliter->second;
		return false;
	}
}

inline void dlcon::EnqJob(tDlJob *todo)
{
	setLockGuard;

	ASSERT(todo);
	ASSERT(todo->m_pStorage->m_nRangeLimit <0
			|| todo->m_pStorage->m_nRangeLimit >= todo->m_pStorage->m_nSizeSeen);

	LOGSTART2("dlcon::EnqJob", todo->m_fileUri.ToURI());

	m_qNewjobs.push_back(tDlJobPtr(todo));
	if (m_wakepipe[1]>=0)
		POKE(m_wakepipe[1]);
}

void dlcon::AddJob(tFileItemPtr m_pItem, 
		const acfg::tRepoData *pBackends, const MYSTD::string & sPatSuffix)
{
	EnqJob(new tDlJob(this, m_pItem, pBackends, sPatSuffix));
}

void dlcon::AddJob(tFileItemPtr m_pItem, tHttpUrl hi)
{
	EnqJob(new tDlJob(this, m_pItem, hi));
}

void dlcon::SignalStop()
{
	LOGSTART("dlcon::SignalStop");
	setLockGuard;

	// stop all activity as soon as possible
	m_bStopASAP=true;
	m_qNewjobs.clear();

	POKE(m_wakepipe[1]);
}

dlcon::~dlcon()
{
	LOGSTART("dlcon::~dlcon, Destroying dlcon");
	checkforceclose(m_wakepipe[0]);
	checkforceclose(m_wakepipe[1]);
}

inline UINT dlcon::ExchangeData(mstring &sErrorMsg, tTcpHandlePtr &con, tDljQueue &qActive)
{
	LOGSTART2("dlcon::ExchangeData",
			"qsize: " << qActive.size() << ", sendbuf size: "
			<< m_sendBuf.size() << ", inbuf size: " << m_inBuf.size());

	fd_set rfds, wfds;
	struct timeval tv;
	int r = 0;
	int fd = con ? con->GetFD() : -1;
	FD_ZERO(&rfds);
	FD_ZERO(&wfds);

	if(qActive.empty())
		m_inBuf.clear(); // dirty buffer from previous connection?

	// no socket operation needed in this case but just process old buffer contents
	bool bReEntered=!m_inBuf.empty();

	loop_again:

	for (;;)
	{
		FD_SET(m_wakepipe[0], &rfds);
		int nMaxFd = m_wakepipe[0];

		if (fd>=0)
		{
			FD_SET(fd, &rfds);
			nMaxFd = MYSTD::max(fd, nMaxFd);

			if (!m_sendBuf.empty())
			{
				ldbg("Needs to send " << m_sendBuf.size() << " bytes");
				FD_SET(fd, &wfds);
			}
		}

		ldbg("select dlcon");
		tv.tv_sec = acfg::nettimeout;
		tv.tv_usec = 0;

		// jump right into data processing but only once
		if(bReEntered)
		{
			bReEntered=false;
			goto proc_data;
		}

		r=select(nMaxFd + 1, &rfds, &wfds, NULL, &tv);

		if (r < 0)
		{
			dbgline;
			if (EINTR == errno)
				continue;
#ifdef MINIBUILD
			string fer("select failed");
#else
			aclog::errnoFmter fer("FAILURE: select, ");
			LOG(fer);
#endif
			sErrorMsg = string("500 Internal malfunction, ") + fer;
			return EFLAG_DO_DISCON|EFLAG_JOB_BROKEN|EFLAG_MIRROR_BROKEN;
		}
		else if (r == 0) // looks like a timeout
		{
			dbgline;
			sErrorMsg = "500 Connection timeout";
			// was there anything to do at all?
			return qActive.empty() ? EFLAG_HINT_SWITCH : (EFLAG_DO_DISCON|EFLAG_JOB_BROKEN);
		}

		if (FD_ISSET(m_wakepipe[0], &rfds))
		{
			dbgline;
			for (int tmp; read(m_wakepipe[0], &tmp, 1) > 0;)
				;
			return EFLAG_HINT_SWITCH;
		}

		if (fd>=0 && FD_ISSET(fd, &wfds))
		{
			FD_CLR(fd, &wfds);

			ldbg("Sending data...\n" << m_sendBuf);
			int s = ::send(fd, m_sendBuf.data(), m_sendBuf.length(), MSG_NOSIGNAL);
			ldbg("Sent " << s << " bytes from " << m_sendBuf.length());
			if (s < 0)
			{
				if (errno != EAGAIN && errno != EINTR)
				{
					sErrorMsg = "502 Peer communication error";
					return EFLAG_JOB_BROKEN|EFLAG_DO_DISCON;
				}
				// else retry later...
			}
			else
				m_sendBuf.drop(s);
		}

		if (fd >=0 && FD_ISSET(fd, &rfds))
		{

			r = m_inBuf.sysread(fd);


#ifdef DISCO_FAILURE
			static int fakeFail=10;
			if( fakeFail-- < 0)
			{
				LOGLVL(LOG_DEBUG, "\n#################\nFAKING A FAILURE\n###########\n");
				r=0;
				fakeFail=10;
				errno = EROFS;
			}
#endif


			if (r <= 0) // there must be data. If no -> disconnected.
			{
				dbgline;

#ifdef MINIBUILD
				sErrorMsg = "502 EPIC FAIL";
#else
				// pickup the error code for later and kill current connection ASAP
				sErrorMsg = aclog::errnoFmter("502 ");
#endif
				return EFLAG_LOST_CON;
			}

			proc_data:

			if(qActive.empty())
			{
				ldbg("FIXME: unexpected data returned?");
				sErrorMsg = "500 Unexpected data";
				return EFLAG_LOST_CON;
			}

			while(!m_inBuf.empty())
			{

				ldbg("Processing job for " << qActive.front()->RemoteUri());
				UINT res = qActive.front()->ProcessIncomming(m_inBuf);
				ldbg(
						"... incoming data processing result: " << res
						<< ", emsg: " << qActive.front()->sErrorMsg);

				if (EFLAG_MORE == res)
					goto loop_again;

				if (EFLAG_DONE & res)
				{
					// just in case that server damaged the last response body
					con->NameLastFile(qActive.front()->m_pStorage->m_sPathRel);

					qActive.pop_front();
					if (EFLAG_DO_DISCON & res)
						return EFLAG_DO_DISCON; // with cleaned flags

					LOG(
							"job finished. Has more? " << qActive.size()
							<< ", remaining data? " << m_inBuf.size());

					if (qActive.empty())
					{
						LOG("Need more work");
						return EFLAG_HINT_SWITCH;
					}

					LOG("Extract more responses");
					continue;
				}
				else // error handling, pass to main loop
				{

					if(EFLAG_KILL_LAST_FILE & res)
						con->KillLastFile();

					setIfNotEmpty(sErrorMsg, qActive.front()->sErrorMsg);
					return res;
				}
			}
			return EFLAG_DONE; // input buffer consumed
		}
	}

	ASSERT(!"Unreachable");
	sErrorMsg = "500 Internal failure";
	return EFLAG_JOB_BROKEN|EFLAG_DO_DISCON;
}

inline void CleanRunning(tDljQueue &inpipe)
{
	for(tDljQueue::iterator it = inpipe.begin(); it!= inpipe.end();)
	{
		if(*it && (**it).m_pStorage
				&& (**it).m_pStorage->GetStatus() >= FIST_DLRECEIVING)
		{
			// someone else is doing it -> drop
			inpipe.erase(it++);
			continue;
		}
		else
			++it;
	}
}

void dlcon::WorkLoop()
{
	LOGSTART("dlcon::WorkLoop");
    string sErrorMsg;
    
	if (!m_inBuf.init(acfg::dlbufsize))
	{
		aclog::err("500 Out of memory");
		return;
	}

	if(m_wakepipe[0]<0 || m_wakepipe[1]<0)
	{
		aclog::err("Error creating pipe file descriptors");
		return;
	}

	tDljQueue inpipe;
	tTcpHandlePtr con;
	UINT loopRes=0;

	bool bStopRequesting=false; // hint to stop adding request headers until the connection is restarted

	int nLostConTolerance=0;
#define MAX_RETRY_USED 4
#define MAX_RETRY_FRESH 2

	while(true) // outer loop: jobs, connection handling
	{

        // init state or transfer loop jumped out, what are the needed actions?
        {
        	setLockGuard;
        	LOG("New jobs: " << m_qNewjobs.size());
        	if(m_bStopASAP)
        	{
        		// ordered to stop but someone might rely on the running job. Keep it running,
        		// if the last user disappears and will cause the termination RSN
        		if(inpipe.size()>1)
        			inpipe.erase(++(inpipe.begin()), inpipe.end());

        		if(inpipe.empty())
        		{
        			if(con)
        				tcpconnect::RecycleIdleConnection(con);

        			return;
        		}
        	}

        	if(m_qNewjobs.empty())
        		goto go_select; // parent will notify RSN

        	if(!con)
        	{
        		// cleanup after the last connection - send buffer, broken jobs, ...
        		m_sendBuf.clear();
        		m_inBuf.clear();

        		bool bUsed=false;
        		bStopRequesting=false;

        		for(tDljQueue::iterator it=m_qNewjobs.begin(); it!=m_qNewjobs.end();)
        		{
        			if(SetupJobConfig(*it, &sErrorMsg))
        				++it;
        			else
        			{
        				setIfNotEmpty2( (**it).sErrorMsg, sErrorMsg,
        						"500 Broken mirror or incorrect configuration");
        				m_qNewjobs.erase(it++);
        			}
        		}
        		if(m_qNewjobs.empty())
        		{
        			LOG("no jobs left, start waiting")
        			goto go_select; // nothing left, might receive new jobs soon
        		}
        		if(acfg::proxy_info.sHost.empty())
        			con = tcpconnect::CreateConnected(m_qNewjobs.front()->GetPeerName(),
        				m_qNewjobs.front()->GetPeerPort(), sErrorMsg,
        				&bUsed, m_qNewjobs.front()->GetConnStateTracker());
        		else
        			con = tcpconnect::CreateConnected(acfg::proxy_info.sHost,
        					acfg::proxy_info.sPort, sErrorMsg, &bUsed,
        					m_qNewjobs.front()->GetConnStateTracker());

        		nLostConTolerance = bUsed ? MAX_RETRY_USED : MAX_RETRY_FRESH;
        		ldbg("connection valid? " << bool(con) << " was fresh? " << !bUsed);

        		if(con)
        		{
        			ldbg("target? [" << con->GetHostname() << "]:" << con->GetPort());
        		}
        		else
        		{
        			BlacklistMirror(m_qNewjobs.front(), sErrorMsg);
        			continue; // try the next backend
        		}
        	}

        	// connection should be stable now, prepare all jobs and/or move to pipeline
        	while(!bStopRequesting && !m_qNewjobs.empty())
        	{
        		tDlJobPtr &cjob = m_qNewjobs.front();

        		bool bGoodConfig = SetupJobConfig(cjob, NULL);

        		ldbg("target: " << cjob->GetPeerName() << " vs " << con->GetHostname()
        				<< ", ports: " << cjob->GetPeerPort() << " vs " << con->GetPort()
        				<< ", good config: " << bGoodConfig);

        		if(!bGoodConfig)
        			continue;

        		if(acfg::proxy_info.sHost.empty()
        				// needs to send them for the connected target host
        				&& ( cjob->GetPeerName() != con->GetHostname()
        				|| cjob->GetPeerPort() != con->GetPort()) )
        		{
        			LOG("host mismatch, stop requesting");
        			bStopRequesting=true;
        			break;
        		}
        		else
        		{
        			cjob->AppendRequest(m_sendBuf, m_sXForwardedFor);
        			LOG("request added to buffer");
        			inpipe.push_back(cjob);
        			m_qNewjobs.pop_front();
        		}
        	}
        }

        go_select:

        if(inpipe.empty() && m_bManualMode)
        {
        	return;
        }

        // either have something to receive or something to send (sooner or later)
		ASSERT( !inpipe.empty() || !bStopRequesting);

         // inner loop: plain communication until something happens. Maybe should use epoll here?
        loopRes=ExchangeData(sErrorMsg, con, inpipe);
        ldbg("loopRes: "<< loopRes);

        // no matter what happened, that stop flag is now irrelevant
        if(inpipe.empty())
        {
        	dbgline;
        	bStopRequesting=false;
        }

        if(0==(loopRes&(EFLAG_DO_DISCON|EFLAG_JOB_BROKEN|EFLAG_LOST_CON|EFLAG_MIRROR_BROKEN))
        		&& con && inpipe.empty()) // no error bits set, not busy -> recycle it when idle
        {
        	dbgline;
			tcpconnect::RecycleIdleConnection(con);
			continue;
		}

        if( (EFLAG_DO_DISCON|EFLAG_LOST_CON) & loopRes)
        {
        	dbgline;
        	con.reset();
        }

		if ((EFLAG_LOST_CON & loopRes) && !inpipe.empty())
		{
			// disconnected by OS... give it a chance, or maybe not...
			if (--nLostConTolerance <= 0)
				BlacklistMirror(inpipe.front(), sErrorMsg);

			// can we resume that job secretly? If no, let the error be forwarded to the user
			if (acfg::vrangeops <= 0 && inpipe.front()->m_pStorage->m_bCheckFreshness)
				loopRes |= EFLAG_JOB_BROKEN;
			else
				inpipe.front()->m_DlState = tDlJob::STATE_REGETHEADER;
		}


        if(loopRes<=EFLAG_DONE)
        {
        	nLostConTolerance=MAX_RETRY_FRESH;
        	continue;
        }

        // regular required post-processing done
        if(EFLAG_HINT_SWITCH == loopRes)
        	continue;

        // ok, now needs to resolve the fatal error situation, push the pipelined back to new, etc.

        if( (EFLAG_MIRROR_BROKEN & loopRes) && !inpipe.empty())
        	BlacklistMirror(inpipe.front(), sErrorMsg);

        if( (EFLAG_JOB_BROKEN & loopRes) && !inpipe.empty())
        {
        	setIfNotEmpty(inpipe.front()->sErrorMsg, sErrorMsg);

        	inpipe.pop_front();

        	if(EFLAG_STORE_COLLISION & loopRes)
        	{
				// stupid situation, both users downloading the same stuff - and most likely in the same order
				// if one downloader runs a step ahead (or usually many steps), drop all items
				// already processed by it and try to continue somewhere else.
				// This way, the overall number of collisions and reconnects is minimized

        		CleanRunning(inpipe); // seriously, I want lambdas
        		setLockGuard;
        		CleanRunning(m_qNewjobs);
        	}

        }


        {
        	setLockGuard;
        	m_qNewjobs.insert(m_qNewjobs.begin(), inpipe.begin(), inpipe.end());
        	inpipe.clear();
        }

	}
}
