# Yahoo
# (theoretically there are no language dependencies)
# supports text, html, images and other files as attachments
# supports non-standard encodings
# supports sending of text mails (attachments don't yet work)
#
# @Requires: 1.1 RC2
# @Version: 0.8.2
#    german texts (with MrPostman 1.2.2+)
# @Version: 0.8.1
#    Fixed: Message with attachment would download attachments of previous message
# @Version: 0.8
#    Fixed: Inboxlink not found for old accounts
# @Version: 0.7.8
#    MVL 050505
#    fixed attachments, if e.g. only one zip.
#    get attachments only once - remove duplicate links.
#    Getting name of attachment language independent.
#    (not tested with old 250MB accounts - mine is now upgraded, too)
# @Version: 0.7.7
#    S_K 050502 - adjusted login for Yahoo 1 GB mailboxes.
#    fixed logoff and attachment functions.
# @Version: 0.7.6
#    FAH 050303 - adjusted login for new Yahoo!
#    compose, delete, and logoff, functions broken
#    MVL 050304 - merged: only some accounts changed
# @Version: 0.7.4
#    determine correct charset also for multipart messages
# @Version: 0.7.3
#    don't even expect a from: header; case insensitive header match
# @Version: 0.7.2
#    don't rely on space after colon in headers
# @Version: 0.7
#    corrected: counting of messages, deletions in multiple folders
# @Version: 0.6
#    support for multiple folders, merged version of siafu_kali
# @Version: 0.5
#    return all headers in msg.headers (only effective for 1.1RC2+)
# @Version: 0.4
#    fixed https, number of messages, pattern for old messages
# ...

sub getInfo
	local(info)
	info.name = "yahoo intl"
	info.authors[0] = "Martin Vlcek <mvlcek@users.sourceforge.net>"
	info.authors[1] = "Siafu Kali <siafu_kali@juno.com>"
	info.authors[2] = "Javier Kohen <jkohen@users.sourceforge.net>"
	info.authors[3] = "Frank0 Brain <fa0_he@users.sourceforge.net>"
	info.version = "0.8.2"
	info.updateService = ""
	info.documentationLink = ""
	info.options["useHttps"].order = 1
	info.options["useHttps"].value = false
	info.options["useHttps"].label = "Use HTTPS"
	info.options["useHttps"].description = "Use HTTPS rather than HTTP for sign on"
	info.options["folders"].order = 2
	info.options["folders"].value = "Inbox, @B@Bulk"
	info.options["folders"].label = "Folders"
	info.options["folders"].description = "The webmail folders to check for retrieval, e.g. Inbox, @B@Bulk"
	info.options["maxNum"].order = 3
	info.options["maxNum"].value = 100
	info.options["maxNum"].label = "Max Number of Messages"
	info.options["maxNum"].description = "The maximum number of messages to check for retrieval"
	info.options["adjustIds"].order = 4
	info.options["adjustIds"].value = false
	info.options["adjustIds"].label = "EXP: Adjust Message Ids"
	info.options["adjustIds"].description = "Remove changing part from message ids (experimental)"
	#--- german texts
	info.options["useHttps"].de.label = "HTTPS verwenden"
	info.options["useHttps"].de.description = "Für das Anmelden HTTPS anstatt HTTP verwenden"
	info.options["folders"].de.label = "Ordner"
	info.options["folders"].de.description = "Webmail Ordner, aus denen Nachrichten heruntergeladen werden sollen, z.B.: Inbox, @B@Bulk"
	info.options["maxNum"].de.label = "Max. Anzahl Nachrichten"
	info.options["maxNum"].de.description = "Die maximale Anzahl von Nachrichten, die überprüft bzw. heruntergeladen werden sollen"
	info.options["adjustIds"].de.label = "EXP: Nachrichten-Ids anpassen"
	info.options["adjustIds"].de.description = "Sich ändernden Teil der Nachrichten-Ids entfernen (experimentell)"
	return info
end

sub getExtensions
	local(extEuro,extAmer,extAsia,extPaci)
	#--- all extensions known currently (2004-08-13)
	extEuro = list("@yahoo.dk","@yahoo.de","@yahoo.es","@yahoo.fr","@yahoo.it","@yahoo.no","@yahoo.se","@yahoo.co.uk","@yahoo.ie")
	extAmer = list("@yahoo.com","@yahoo.ca","@yahoo.com.mx","@yahoo.com.ar","@yahoo.com.br")
	extAsia = list("@yahoo.com.cn","@yahoo.com.hk","@yahoo.co.in","@yahoo.co.jp","@yahoo.co.kr","@yahoo.com.sg","@yahoo.com.tw")
	extPaci = list("@yahoo.com.au","@yahoo.co.nz")
	return join(extEuro,extAmer,extAsia,extPaci)
end

sub login(username,password)
	local(loginurl,params,status,page,found,loginaction)
	local(uFound,uValue,isNew,welcomeurl)
	#debug(true)
	#--- set some basic settings to allow MrPostman doing some conversions, ...
	loginurl = iif(info.options["useHttps"].value,"https","http") & "://login.yahoo.com/config/login"
	#--- sizes and dates seem to be independent of the language
	size.locale = "en"
	size.unit = map("b",1,"k",1024,"m",1024*1024,"M",1024*1024)

	#*** Javier Kohen from bugs
	#*** Yahoo! sends some invalid cookies and it doesn't seem to like
	#*** multiple cookie headers, which the strict mode prevents.
	setcookiemode("COMPATIBILITY")
	setstrictmode(true)

	#*** FAH 050303 - Yahoo Mail has changed login procedures and links
	#*** I grab a raw, unsecured page, so as to avoid redirects
	#*** Then I parse out it's .u value, which seems to be a needed parameter
	#*** An example:
	#***     <input type=hidden name=".u" value="cfqu4jp12e9i9" >
	status,page = get("http://mail.yahoo.com")
	uFound,uValue = match(page,"name=.u value=\"([^\"]*)")

	#--- let's log in:
	found,username = match(username,"^(.*?)@")
   	params[".tries"] = "1"
	params[".src"] = "ym"
	params[".us"] = "ym"
	params[".intl"] = "us"
	if (uFound)
		params[".u"] = uValue
	end
	params["login"] = username
	params["passwd"] = password
	status,page = get(loginurl,params)
	#--- check for redirect (if we logged in with https)
	found,redirecturl = match(page,"replace\(\"([^\"]*login[^\"]*)\"\)")
	if (found)
		status,page = get(redirecturl)
	end

	#*** FAH 050303 - The inbox link has changed for some accounts!
	#*** An example:
	#***    var gInboxPage = "http://us.f514.mail.yahoo.com/ym/ShowFolder?YY=81855&box=Inbox&YN=1";
	#--- find the inbox link
	#isNew,inboxlink = match(page,"gInboxPage = \"([^\"]*ShowFolder\?[^\"]*)")
	isNew,welcomeurl = match(page,"gWelcomePage = \"([^\"]*)")
	if (isNew)
		#--- derive inbox from welcome page
		inboxlink = replace(welcomeurl, "welcome", "ShowFolder") & "&box=Inbox"
		#--- find the message link - derive it from inbox link
		#found,msglink = match(page,"gInboxPage = \"([^\"]*ShowFolder\?[^&]*&box=)")
		msglink = replace(inboxlink,"Inbox","")
		#--- get compose and logout links from welcome page?
		#found,welcomeurl = match(page,"gWelcomePage = \"([^\"]*)")
		status,page = get(welcomeurl)
		#--- probably fails - who can fix it?
		found,composelink = match(page,"window.location='([^']*Compose\?[^']*)'")
		#found,logoutlink = match(page,"href=\"([^\"]*Logout\?[^\"]*)\"")
		logoutlink ="http://login.yahoo.com/config/login?logout=1"
	end
	if (!isNew)
		#*** Most accounts did not change (?)
		found,inboxlink = match(page,"location='([^\"']*ShowFolder\?rb=Inbox[^\"']*)'")
		#--- if there is none, we are probably not logged in
		returnif(!found,false)
		#--- find the message link - derive it from inbox link
		found,msglink = match(page,"location='([^\"]*ShowFolder\?rb=)[^\"]*'")
		# other important links to remember
		found,composelink = match(page,"window.location='([^']*Compose\?[^']*)'")
		found,logoutlink = match(page,"href=\"([^\"]*Logout\?[^\"]*)\"")
	end
	setbaseurl(inboxlink)
	folders = split(info.options["folders"].value, "\s*,\s*")
	#debug(false)
	return true
end

sub loginForReceive(username,password)
	local(ok,folder)
	ok = login(username,password)
	returnif(!ok,false)
	#--- get messages for each folder
	msgs = list()
	foreach(folders,folder)
		msgs = join(msgs,getMessageIds(folder))
	end
	return true
end

sub getMessageIds(folder)
	local(status,page,found,msglist,msgpat,numfound,link,msgs1,msgs2,part1,part2,part3)
	status,page = get(msglink & urlencode(folder))
	found,msglist,dellink[folder] = match(page,"(<form\s+name=messageList[^>]*action=\"([^\"]*)\".*?</form>)")
	delparams[folder] = getformfields(msglist)
	#--- corrected according to Srinivasan:
	msgpat = "href=\"([^\"]*ShowLetter\?MsgId=([^&]+)&[^\"]*&box=([^\"]*))\".*?<td.*?<td[^>]*>(\d+)([a-zA-Z]*)<"
	msgs1 = list()
	numfound,msgs1[].link,msgs1[].idyahoo,msgs1[].folder,msgs1[].size.value,msgs1[].size.unit = matchall(page,msgpat)
	#--- further pages:
	found,nextpagelink = match(page,"href=\"([^\"]*ShowFolder[^\"]*next=1[^\"]*)\"")
	while (found and size(msgs) + size(msgs1) <= integer(info.options["maxNum"].value))
		status,page = get(nextpagelink)
		msgs2 = list()
		numfound,msgs2[].link,msgs2[].idyahoo,msgs2[].folder,msgs2[].size.value,msgs2[].size.unit = matchall(page,msgpat)
		msgs1 = join(msgs1,msgs2)
		found,nextpagelink = match(page,"href=\"([^\"]*ShowFolder[^\"]*next=1[^\"]*)\"")
	end
	foreach(msgs1,msg)
		msg.id = msg.idyahoo
		#--- patch according to alanrf:
		#--- (the changing part is removed, but the original kept for message manipulations)
		if (info.options["adjustIds"].value)
			found,part1,part2,part3 = match(msg.idyahoo,"^(\d+_)(\d+_\d+_)(.*)$")
			msg.id = part1 & part3
		end
		msg.folder = replace(msg.folder,"#.*$","")
	end
	info("Checking Folder: " & folder & " (" & size(msgs1) & ") messages")
	return msgs1
end

sub receive(msg)
	local(status,page,found,numfound,mimtyp,part1,headers)
	local(attachpat,links,inlineattachments,attachment)
	#--- yahoo allows us to read the original headers:
	status,page = get("/ym/ShowLetter?box=" & msg.folder & "&MsgId=" & msg.idyahoo & "&bodyPart=HEADER")
	found,msg.from    = match(page,"^From:\s*(.*?)$",true)
	#if(!found)
	#	error("Invalid page structure for message '" & msg.id & "'")
	#end
	found,msg.to      = match(page,"^To:\s*(.*?)\\n(?:\\n|[^\s])",true)
	found,msg.cc      = match(page,"^Cc:\s*(.*?)\\n(?:\\n|[^\s])",true)
	found,msg.bcc     = match(page,"^Bcc:\s*(.*?)\\n(?:\\n|[^\s])",true)
	found,msg.date    = match(page,"^Date:\s*(.*?)$",true)
	found,msg.subject = match(page,"^Subject:\s*(.*?)\\n(?:\\n|[^\s])",true)
	found,msg.charset = match(page,"^Content-Type:.*?charset=([^;\s]*)[;\s]",true)
	found,mimtyp      = match(page,"^Content-Type:\s*([^\s]+)",true)
	if (find(mimtyp,"^multipart",true)) 
		#--- get charset from first body part - should work in most cases
		status,part1 = get("/ym/ShowLetter?box=" & msg.folder & "&MsgId=" & msg.idyahoo & "&bodyPart=1")
		found,msg.charset = match(part1,"^Content-Type:.*?charset=([^;\s]*)[;\s]",true)
	end
	#--- get all headers - only effective in 1.1RC2+
	numfound,headers[].name,headers[].value = matchall(page&"\n\n","^(\w[\w\d-]*):\s*(.*?)\\n(?=\\n|[^\s])")
	msg.headers = map(headers[].name,headers[].value,true)
	#--- Add Folder header for email client filtering
	msg.headers = merge(msg.headers,map("Folder",msg.folder))
	#--- it's always an html mail
	msg.mimetype = "text/html"
	#--- now get the actual message (we use the encoding of the header instead of that of the page):
	status,page = get(msg.link,map(),map(),msg.charset)
	found,msg.text = match(page,"<div id=message>(.*)</div>\s*<!-- END TOC -->")
	#-- get attachments
	msg.attachments = list()
	if (find(page,"<div\s+class=\"splittercontent\"")) 
		#--- there are attachments
		attachpat = "<a href=\"([^\"]*ShowLetter\?[^\"]*bodyPart=[^\"]*)\""
		numfound,links[] = matchall(page,attachpat,true)
		#--- remove duplicates
		msg.attachments[].link = values(map(links,links))
		#--- link will go to another page and is not yet the attachment
		foreach(msg.attachments,attachment)
			status,page = get(attachment.link)
			#--- independent of language:
			found,attachment.text = match(page,"<table class=\"virusscanresult\".*?<td valign=\"top\">([^<]*)<")
			found,attachment.link = match(page,"window.location.href='([^']*download=1[^']*)'")
		end
		found,msg.text = match(msg.text,"(.*?)<div\s+class=\"splittercontent\"")
	end
	if (find(msg.text,"<div\s+class=\"messageinfo separator\""))
		#--- there are inline attachments
		attachpat = "href=\"([^\"]*ShowLetter/[^\"]*filename=([^&\"]*)&[^\"]*download=1[^\"]*)\""
		numfound,inlineattachments[].link,inlineattachments[].text = matchall(msg.text,attachpat,true)
		#--- the names of the attachment are part of the url, thus let's decode them:
		foreach(inlineattachments,attachment)
			attachment.text = urldecode(attachment.text)
		end
		msg.attachments = join(msg.attachments,inlineattachments)
		found,msg.text = match(msg.text,"\A(.*?)<div\s+class=\"messageinfo separator\"")
	end
	#--- envelope message in html:
	msg.text = "<html><body>\n" & msg.text & "\n</body></html>"
	#--- find inline pictures in text:
	inlineattachments = list()
	attachpat = "<img[^>]*src=\"([^\"]*bodyPart=([^\"&]+)[^\"]*)\""
	numfound,inlineattachments[].link,inlineattachments[].id = matchall(msg.text,attachpat,true)
	foreach(inlineattachments,attachment)
		attachment.text = attachment.id
		msg.text = replace(msg.text,"src=\"[^\"]*bodyPart="&attachment.id&"[^\"]*\"","src=\"cid:"&attachment.id&"\"")
	end
	msg.attachments = join(msg.attachments,inlineattachments)
	sleep(1000)
	return msg
end

sub delete(msgs)
	local(i,status,page,mydelparams,mydellink,msg)
	foreach(folders,folder)
		mydellink = dellink[folder]
		mydelparams = delparams[folder]
		i = 0
		foreach(msgs,msg)
			if (msg.folder == urlencode(folder))
				mydelparams["Mid#" & i] = msg.idyahoo
				i = i+1
			end
		end
		if (i > 0)
			mydelparams["DEL"] = "Delete"
			status,page = post(mydellink,mydelparams)
		end
	end
	return true
end

sub loginForSend(username,password)
	return login(username,password)
end

sub send(msg)
	local(composeformpattern,attachformpattern)
	local(status,page,found,action,content,params,fileparams,i)
	composeformpattern = "<form[^>]*name=\"Compose\"[^>]*action=\"([^\"]*)\"[^>]*>(.*?)</form"
	attachformpattern = "<form[^>]*name=\"Attachments\"[^>]*action=\"([^\"]*)\"[^>]*>(.*?)</form"
	#--- get the compose page
	status,page = get(composelink)
	if (size(msg.attachments) > 0)
		warning("Attachments don't work, but why???")
		#--- go to the attachment page
		found,action,content = match(page,composeformpattern)
		params = getformfields(content)
		params["ATT"] = "1"
		status,page,headers = post(action,params)
		#--- post all attachments
		found,action,content = match(page,attachformpattern)
		params = getformfields(content)
		i = 0
		fileparams = map()
		foreach(msg.attachments,att)
			fileparams["userFile" & i] = att.link
			i = i + 1
		end
		#while (i < 50) 
		#	fileparams["userFile" & i] = ""
		#	i = i + 1
		#end
		params["UPL"] = "Attach Files"
		status,page = postmultipart(action,fileparams,params)
		#--- finished with attachments - back to compose page
		found,action,content = match(page,composeformpattern)
		params = getformfields(content)
		params["FName"] = ""
		status,page = post(action,params)
	end
	#--- set msg fields and post message
	found,action,content = match(page,composeformpattern)
	params = getformfields(content)
	params["To"] = msg.to
	params["Cc"] = msg.cc
	params["Bcc"] = msg.bcc
	params["Subj"] = msg.subject
	params["Body"] = msg.text
	params["SEND"] = "1"
	status,page = post(action,params)
	return (status == 200)
end

sub logout
	get(trim(logoutlink))
end