using System;
using System.IO;
using System.Collections;
using System.Data;
using System.Web;
using System.Net;
using System.Text.RegularExpressions;
using System.Text;
using Orciid.Media;


namespace Orciid.Core
{
	/// <summary>
	/// Nasa Image eXchange remote collection
	/// </summary>
	/// <remarks>
	/// This collection uses screen scraping to give access to NASA's NIX collection
	/// within MDID.
	/// </remarks>
	public class NIXRemoteCollection:
		RemoteCollection
	{
		const int MAXRESULTS = 100;
		
		/// <summary>
		/// Default constructor
		/// </summary>
		/// <remarks>
		/// Default constructor
		/// </remarks>
		public NIXRemoteCollection():
			base()
		{
		}

		/// <summary>
		/// Constructor
		/// </summary>
		/// <remarks>
		/// Internal constructor
		/// </remarks>
		internal NIXRemoteCollection(bool init):
			base(init)
		{
		}

		/// <summary>
		/// Collection type
		/// </summary>
		/// <remarks>
		/// Returns the collection type identifier, which is <c>N</c>.
		/// </remarks>
		/// <returns><c>N</c></returns>
		protected override char GetRemoteCollectionType()
		{
			return 'N';
		}

		/// <summary>
		/// Service information
		/// </summary>
		/// <remarks>
		/// Each remote collection keeps track of service information, e.g. information
		/// for the current login session on the remote server. For this
		/// type of remote collection, this is the URL of the search form which includes
		/// a session identifier.
		/// </remarks>
		/// <returns></returns>
		protected override object GetServiceInfo()
		{
			return this.GetSearchFormURL();
		}

		private void CreateCollectionFields()
		{
			if (this.fields.Count > 0)
				return;

			Field field;

			field = new Field();
			field.Label = "Title";
			field.Name = "Title";
			field.DCElement = "Title";
			field.Type = FieldType.Text;
			field.Searchable = false;
			field.KeywordSearchable = false;
			field.Sortable = true;
			field.Browsable = false;
			field.ShortView = DisplayMode.All;
			field.MediumView = DisplayMode.All;
			field.LongView = DisplayMode.All;
			AddField(field);

			field = new Field();
			field.Label = "Identifier";
			field.Name = "ID";
			field.DCElement = "Identifier";
			field.Type = FieldType.Text;
			field.Searchable = false;
			field.KeywordSearchable = false;
			field.Sortable = true;
			field.Browsable = false;
			field.ShortView = DisplayMode.None;
			field.MediumView = DisplayMode.All;
			field.LongView = DisplayMode.All;
			AddField(field);

			field = new Field();
			field.Label = "Description";
			field.Name = "Description";
			field.DCElement = "Description";
			field.Type = FieldType.Text;
			field.Searchable = false;
			field.KeywordSearchable = false;
			field.Sortable = false;
			field.Browsable = false;
			field.ShortView = DisplayMode.None;
			field.MediumView = DisplayMode.All;
			field.LongView = DisplayMode.All;
			AddField(field);

			field = new Field();
			field.Label = "Date";
			field.Name = "Date";
			field.DCElement = "Date";
			field.Type = FieldType.Text;
			field.Searchable = false;
			field.KeywordSearchable = false;
			field.Sortable = false;
			field.Browsable = false;
			field.ShortView = DisplayMode.All;
			field.MediumView = DisplayMode.All;
			field.LongView = DisplayMode.All;
			AddField(field);

			field = new Field();
			field.Label = "Credit";
			field.Name = "Credit";
			field.DCElement = "Rights";
			field.Type = FieldType.Text;
			field.Searchable = false;
			field.KeywordSearchable = false;
			field.Sortable = false;
			field.Browsable = false;
			field.ShortView = DisplayMode.All;
			field.MediumView = DisplayMode.All;
			field.LongView = DisplayMode.All;
			AddField(field);

			field = new Field();
			field.Label = "Image URL";
			field.Name = "imageurl";
			field.DCElement = null;
			field.Type = FieldType.Text;
			field.Searchable = false;
			field.KeywordSearchable = false;
			field.Sortable = false;
			field.Browsable = false;
			field.ShortView = DisplayMode.None;
			field.MediumView = DisplayMode.None;
			field.LongView = DisplayMode.All;
			AddField(field);
		}

		/// <summary>
		/// Number of images in collection
		/// </summary>
		/// <remarks>
		/// Since the NIX collection does not provide an image count on the web site,
		/// the actual image count is unknown.  This is indicated by returning <c>-1</c>.
		/// </remarks>
		/// <value>
		/// <c>-1</c>
		/// </value>
		public override int ImageCount
		{
			get
			{
				return -1;
			}
		}

		/// <summary>
		/// Indicates if record history is kept
		/// </summary>
		/// <remarks>
		/// If this property is <c>true</c>, any change to an image record will cause the
		/// original values to be written to a record history table in the database.
		/// For remote collections, this property is <c>false</c>.
		/// </remarks>
		/// <value><c>false</c></value>
		public override bool HasRecordHistory
		{
			get
			{
				return false;
			}
		}

		internal override ImageIdentifier[] Search(SearchCondition[] conditions, SearchEngine engine)
		{
			try
			{
				string keywords = "";
				foreach (SearchCondition cond in conditions)
					if (cond is KeywordCondition)
						keywords += (keywords.Length > 0 ? " " : "") + ((KeywordCondition)cond).Keyword;
				if (keywords.Length == 0)
					return null;
				
				string searchformurl = GetSearchFormURL();
				using (DBConnection conn = DBConnector.GetConnection())
				{
					string[] result = SearchForKeyword(conn, searchformurl, keywords);
					if (result == null || result.Length == 0)
						return null;

					// build list of local image ids
					Query query = new Query(conn,
						@"SELECT ID FROM Images 
						WHERE CollectionID={collectionid} AND RemoteID IN {remoteids}");
					query.AddParam("collectionid", this.id);
					query.AddParam("remoteids", result);
					int[] localids = conn.TableToArray(conn.SelectQuery(query));
					ImageIdentifier[] iids = new ImageIdentifier[localids.Length];
					for (int i = 0; i < localids.Length; i++)
						iids[i] = new ImageIdentifier(localids[i], this.id);
					return iids;
				}
			}
			catch (Exception ex)
			{
				TransactionLog.Instance.AddException("NIX remote search failed", 
					String.Format("CollectionID={0}", this.id),
					ex);
				return null;
			}
		}

		/// <summary>
		/// Finish downloading image record stubs
		/// </summary>
		/// <remarks>
		/// This collection initially only creates image record stubs for each 
		/// cached image, since a complete image record requires an extra web
		/// request.  This method retrieves and populates the complete image
		/// record for all existing stubs.
		/// </remarks>
		/// <param name="imageids">A list of image records to complete if they
		/// are still stubs.</param>
		public override void CacheStubImageRecords(ArrayList imageids)
		{
			ArrayList ids = new ArrayList();
			foreach (ImageIdentifier id in imageids)
				if (id.CollectionID == this.id && !ids.Contains(id.ID))
					ids.Add(id.ID);

			if (ids.Count == 0)
				return;

			SortedList remoteids = new SortedList();

			using (DBConnection conn = DBConnector.GetConnection())
			{
				Query query = new Query(conn,
					@"SELECT ID,RemoteID FROM Images " +
					"WHERE CollectionID={collectionid} AND ID IN {ids} AND " +
					"RecordStatus&{status}={status}");
				query.AddParam("collectionid", this.id);
				query.AddParam("ids", ids);
				query.AddParam("status", ImageRecordStatus.StubOnly);
				DataTable table = conn.SelectQuery(query);
				foreach (DataRow row in table.Rows)
					remoteids.Add(
						new ImageIdentifier(conn.DataToInt(row["ID"], 0), this.id), 
						conn.DataToString(row["RemoteID"]));
			}

			if (remoteids.Count == 0)
				return;

			ArrayList images = GetImagesByID(false, new ArrayList(remoteids.Keys), false);

			if (images == null)
				return;

			string searchformurl = GetSearchFormURL();

			foreach (Image image in images)
				CacheImageRecord(searchformurl, (string)remoteids[image.ID], image);
		}

		private void CacheImageRecord(string searchformurl, string remoteid, Image updateimage)
		{
			Hashtable result = GetRecordInfo(searchformurl, remoteid);
			if (result == null)
				return;

			Image image = updateimage;
			if (image == null)
			{
				image = new Image(new ImageIdentifier(0, id), null);
				image.createddate = DateTime.Now;
			}
			image.modifieddate = image.createddate;
			image.cacheduntil = DateTime.Now + TimeSpan.FromDays(30);
			image.expires = DateTime.Now + TimeSpan.FromDays(90);
			image.remoteid = remoteid;
			image.recordstatus = ImageRecordStatus.Ok;

			// build field data records myself to circumvent access restrictions
			// this is not very elegant
			Hashtable datafields = new Hashtable();
			foreach (string key in result.Keys)
			{
				FieldData fd = null;
				Field f = this.GetField(key);
				if (f != null)
				{
					if (datafields.ContainsKey(f))
						fd = (FieldData)datafields[f];
					else
					{
						fd = FieldData.CreateFieldData(null, f);
						datafields.Add(f, fd);
					}
				}
				if (fd != null)
				{
					object o = result[key];
					if (o is ArrayList)
						foreach (string s in (ArrayList)o)
							fd.Add(s);
					else
						fd.Add((string)o);
				}
			}
			foreach (Field f in this.GetFields())
			{
				if (datafields.ContainsKey(f))
					image.SetFieldData(f, (FieldData)datafields[f]);
				else
				{
					FieldData blank = FieldData.CreateFieldData(null, f);
					blank.MarkModified();
					image.SetFieldData(f, blank);
				}
			}

			// update the image without checking access restrictions and with force update
			image.Update(false, true);
		}

		private bool BackgroundCacheImageRecord(BackgroundTask task, object info)
		{
			string searchformurl = (string)task.Data["searchformurl"];
			string remoteid = (string)task.Data["remoteid"];

			Image updateimage = null;
			using (DBConnection conn = DBConnector.GetConnection())
			{													   
				Query query = new Query(conn,
					@"SELECT ID,CachedUntil,RecordStatus FROM Images 
					WHERE CollectionID={collectionid} AND RemoteID={remoteid}");
				query.AddParam("collectionid", this.id);
				query.AddParam("remoteid", remoteid);
				DataTable table = conn.SelectQuery(query);
				if (table.Rows.Count == 1)
				{
					ArrayList temp = new ArrayList(1);
					temp.Add(new ImageIdentifier(conn.DataToInt(table.Rows[0]["ID"], 0), this.id));
					ArrayList result = this.GetImagesByID(false, temp, false);
					if (result.Count == 1)						
						updateimage = (Image)result[0];
				}			
			}
			// only cache image if it has not been cached already, if it needs recaching, or
			// if only a stub is available
			if (updateimage == null || 
				updateimage.cacheduntil <= DateTime.Now ||
				((updateimage.recordstatus & ImageRecordStatus.StubOnly) == ImageRecordStatus.StubOnly))
				CacheImageRecord(searchformurl, remoteid, updateimage);
			return true;
		}
		
		/// <summary>
		/// Cache image records
		/// </summary>
		/// <remarks>
		/// This method caches the specified image records
		/// </remarks>
		/// <param name="conn">An open database connection</param>
		/// <param name="oinfo">A valid service info object (see <see cref="GetServiceInfo"/>)</param>
		/// <param name="remoteids">The identifiers of the remote images to cache</param>
		protected override void CacheImageRecords(DBConnection conn, object oinfo, string[] remoteids)
		{
			CreateCollectionFields();
			string searchformurl = (string)oinfo;
			ArrayList missing = new ArrayList(remoteids);
			ArrayList update = new ArrayList();

			// find locally cached images
			Query query = new Query(conn,
				@"SELECT ID,RemoteID,RecordStatus,CachedUntil FROM Images 
				WHERE CollectionID={collectionid} AND RemoteID IN {remoteids}");
			query.AddParam("collectionid", this.id);
			query.AddParam("remoteids", remoteids);

			DataTable table = conn.SelectQuery(query);
		
			// create list of images not cached already or where cache expired
			foreach (DataRow row in table.Rows)
			{
				string rid = conn.DataToString(row["RemoteID"]);
				int lid = conn.DataToInt(row["ID"], 0);
				ImageRecordStatus rs = (ImageRecordStatus)conn.DataToInt(row["RecordStatus"], 0);
				DateTime cacheduntil = conn.DataToDateTime(row["CachedUntil"]);
				if (cacheduntil > DateTime.Now && rs == ImageRecordStatus.Ok)
					missing.Remove(rid);
				else
					update.Add(new ImageIdentifier(lid, this.id));
			}

			// download missing images
			if (missing.Count > 0)
			{
				ArrayList updatingimages = 
					(update.Count > 0 ? this.GetImagesByID(false, update) : null);
				foreach (string remoteid in missing)
				{
					Image updateimage = null;
					if (updatingimages != null)
						foreach (Image i in updatingimages)
							if (i.remoteid == remoteid)
							{
								updateimage = i;
								break;
							}
					CacheImageRecord(searchformurl, remoteid, updateimage);
				}
			}
		}

		/// <summary>
		/// Collection Type
		/// </summary>
		/// <remarks>
		/// This property is read-only.
		/// </remarks>
		/// <value>
		/// Always <see cref="CollectionType.Other"/>.
		/// </value>
		public override CollectionType Type
		{
			get
			{
				return CollectionType.Other;
			}
		}

		/// <summary>
		/// Remote server
		/// </summary>
		/// <remarks>
		/// The remote server this collection connects to
		/// </remarks>
		/// <value>
		/// <c>http://nix.nasa.gov/</c>
		/// </value>
		public override string RemoteServer
		{
			get
			{
				return "http://nix.nasa.gov/";
			}
		}

		/// <summary>
		/// Precache image
		/// </summary>
		/// <remarks>
		/// Caches an image file before it actually has been requested for display, to significantly
		/// speed up the display lateron.
		/// </remarks>
		/// <param name="image">The image to precache</param>
		public override void PrecacheImage(Image image)
		{
			FieldData fd = image["imageurl"];
			if (fd != null && fd.Count > 0)
			{
				string host = new Uri(fd[fd.Count - 1]).Host;
				PrecacheImageFile(image.Resource, ImageSize.Full, host);
			}
		}

		private void DownsizeFile(string resource, string file, ImageSize targetformat)
		{
			SetResourceData(resource, targetformat, file);
		}

		/// <summary>
		/// Cache image file
		/// </summary>
		/// <remarks>
		/// Download and cache an image resource.  If the requested format is smaller
		/// than an already cached format, instead of downloading the requested format
		/// is created by downsizing the cached format.
		/// </remarks>
		/// <param name="resource">The image resource to cache</param>
		/// <param name="format">The size of the image to cache</param>
		protected override void CacheImageFile(string resource, ImageSize format)
		{
			Image image = null;
			FieldData fd;
			string file = this.MapResource(resource, format);
			if (File.Exists(file))
				return;

			// check if bigger file already exists, if so, downsize that

			string fullfile = this.MapResource(resource, ImageSize.Full);
			string mediumfile = this.MapResource(resource, ImageSize.Medium);

			bool fullexists = File.Exists(fullfile);
			bool mediumexists = File.Exists(mediumfile);

			if (fullexists)
			{
				DownsizeFile(resource, fullfile, format);
				return;
			}

			if (format == ImageSize.Thumbnail && mediumexists)
			{
				DownsizeFile(resource, mediumfile, ImageSize.Thumbnail);
				return;
			}

			// no usable local image found, need to download

			if (image == null)
			{
				ArrayList images = this.GetImagesByResource(resource);
				if (images != null && images.Count == 1)
					image = (Image)images[0];
				else
					return;
			}

			fd = image["imageurl"];
			if (fd == null || fd.Count == 0)
				// no URL to remote image available
				return;

			string imageurl;
			if (format == ImageSize.Thumbnail)
				imageurl = fd[0];
			else if ((image.recordstatus & ImageRecordStatus.StubOnly) == ImageRecordStatus.StubOnly)
				return; // only have image record stub, nothing but thumbnail available
			else
				imageurl = fd[fd.Count - 1];

			try
			{
				HttpWebRequest req = (HttpWebRequest)HttpWebRequest.Create(imageurl);
				WebResponse response = req.GetResponse();
				Stream stream = response.GetResponseStream();

				if (format == ImageSize.Medium)
				{
					// we downloaded the biggest available, so set the full image and
					// then downsample to medium; this saves us a download when the
					// full image is requested
					this.SetResourceData(resource, ImageSize.Full, stream, response.ContentType);
					DownsizeFile(resource, fullfile, ImageSize.Medium);
				}
				else
					this.SetResourceData(resource, format, stream, response.ContentType);
				stream.Close();
				
			}
			catch (Exception ex)
			{
				TransactionLog.Instance.AddException("Remote image retrieval failed",
					String.Format("CollectionID={0}", this.id),
					ex);
				return;
			}				
		}

		private string GetSearchFormURL()
		{
			HttpWebRequest req = (HttpWebRequest)HttpWebRequest.Create("http://nix.nasa.gov/");
			WebResponse response = req.GetResponse();
			StreamReader reader = new StreamReader(response.GetResponseStream());
			string content = reader.ReadToEnd();
			reader.Close();

			Regex regex = new Regex("<form\\s[^>]*action=\"([^\"]+)\"", 
				RegexOptions.IgnoreCase);
			Match match = regex.Match(content);
			if (match.Success)
				return match.Groups[1].Value;
			else
				return null;
		}

		// returns -1 if number unknown
		private int GetNumberOfMatches(string content)
		{
			if (content.IndexOf("No matches found.") >= 0)
				return 0;
			Regex regex = new Regex("Found <span class=\"bold\">(\\d+)</span>\\s+matches.");
			Match match = regex.Match(content);
			if (match.Success)
				return Int32.Parse(match.Groups[1].Value);
			else
				return -1;
		}

		private Hashtable FindResultsOnPage(string content)
		{
			Regex regex = new Regex("<img src=\"([^\"]+)\".*?" +
				"([^>]+)<\\/span><br \\/>\\s*<a href=\"[^?]+/info[^?]*\\?([^\"]+)\"", 
				RegexOptions.IgnoreCase | RegexOptions.Singleline);
			MatchCollection matches = regex.Matches(content);
			Hashtable result = new Hashtable();
			foreach (Match m in matches)
				result.Add(m.Groups[3].Value, new string[] { m.Groups[1].Value, m.Groups[2].Value.Trim() });
			return result;
		}

		private void CreateImageRecordStubs(DBConnection conn, string searchformurl, Hashtable results)
		{
			CreateCollectionFields();
			Hashtable missing = new Hashtable();
			foreach (string s in results.Keys)
				missing[s] = -1;

			// find locally cached images (exclude image stubs)
			Query query = new Query(conn,
				@"SELECT RemoteID,RecordStatus FROM Images 
				WHERE CollectionID={collectionid} AND RemoteID IN {remoteids}");
			query.AddParam("collectionid", this.id);
			query.AddParam("remoteids", results.Keys);
			DataTable table = conn.SelectQuery(query);

			// create list of images not cached already
			foreach (DataRow row in table.Rows)
			{
				string remoteid = conn.DataToString(row["RemoteID"]);
				ImageRecordStatus recordstatus = (ImageRecordStatus)conn.DataToInt(row["RecordStatus"], 0);
				if (recordstatus == ImageRecordStatus.Ok)
					missing.Remove(remoteid);
				else
					missing[remoteid] = 1;
			}

			foreach (string remoteid in missing.Keys)
			// create image record stubs for missing images
			{
				if ((int)missing[remoteid] == -1)
				{
					Hashtable info = new Hashtable();
					string[] s = (string[])results[remoteid];
					info.Add("imageurl", s[0]);
					info.Add("Title", s[1]);
					info.Add("Identifier", remoteid);

					Image image = new Image(new ImageIdentifier(0, id), null);
					image.createddate = DateTime.Now;
					image.modifieddate = image.createddate;
					image.cacheduntil = DateTime.Now + TimeSpan.FromDays(1);
					image.expires = DateTime.Now + TimeSpan.FromDays(1);
					image.remoteid = remoteid;

					// build field data records myself to circumvent access restrictions
					// this is not very elegant
					Hashtable datafields = new Hashtable();
				
					foreach (string key in info.Keys)
					{
						FieldData fd = null;
						Field f = this.GetField(key);
						if (f != null)
						{
							if (datafields.ContainsKey(f))
								fd = (FieldData)datafields[f];
							else
							{
								fd = FieldData.CreateFieldData(null, f);
								datafields.Add(f, fd);
							}
						}
						if (fd != null)
						{
							object o = info[key];
							if (o is ArrayList)
								foreach (string v in (ArrayList)o)
									fd.Add(v);
							else
								fd.Add((string)o);
						}
					}
					foreach (Field f in datafields.Keys)
						image.SetFieldData(f, (FieldData)datafields[f]);

					image.recordstatus = ImageRecordStatus.StubOnly;
					// update the image without checking access restrictions and with force update
					image.Update(false, true);
				}

				// schedule task to download full record in the background
				BackgroundTask task = 
					new BackgroundTask(String.Format("NIX Collection {0} image record {1}",
					this.id, remoteid), 
					1, 
					"nixremotecollection");
				task.OnRun += new BackgroundTaskDelegate(BackgroundCacheImageRecord);
				task.Data["remoteid"] = remoteid;
				task.Data["searchformurl"] = searchformurl;
				task.Queue();
			}
		}

		private string[] SearchForKeyword(DBConnection conn, string searchformurl, string keyword)
		{
			string postdata = "submit=SEARCH&k=H&t=IMAGE&h=100&n=5&qa=" + HttpUtility.UrlEncode(keyword);
			ASCIIEncoding encoding = new ASCIIEncoding();
			byte[] postdatabytes = encoding.GetBytes(postdata);

			HttpWebRequest req = (HttpWebRequest)HttpWebRequest.Create(searchformurl);
			req.Method = "POST";
			req.ContentType = "application/x-www-form-urlencoded";
			req.ContentLength = postdata.Length;
			Stream requeststream = req.GetRequestStream();
			requeststream.Write(postdatabytes, 0, postdatabytes.Length);
			requeststream.Close();

			WebResponse response = req.GetResponse();
			StreamReader reader = new StreamReader(response.GetResponseStream());
			string content = reader.ReadToEnd();
			reader.Close();

			int matches = GetNumberOfMatches(content);
			if (matches == 0)
				return null;

			Hashtable resultsonpage = FindResultsOnPage(content);
			ArrayList result = new ArrayList();
			result.AddRange(resultsonpage.Keys);
			CreateImageRecordStubs(conn, searchformurl, resultsonpage);

			if (result.Count < MAXRESULTS)
			{
				// not enough results yet, look for more result pages
				Regex regex = new Regex("<a href=\"([^?]+\\?p=\\d+)\"");
				foreach (Match m in regex.Matches(content))
				{
					string pageurl = m.Groups[1].Value;
					req = (HttpWebRequest)HttpWebRequest.Create(pageurl);
					response = req.GetResponse();
					reader = new StreamReader(response.GetResponseStream());
					content = reader.ReadToEnd();
					reader.Close();
					resultsonpage = FindResultsOnPage(content);
					result.AddRange(resultsonpage.Keys);
					CreateImageRecordStubs(conn, searchformurl, resultsonpage);
					if (result.Count >= MAXRESULTS)
						break;
				}
			}
			return (result.Count == 0 ? null : (string[])result.ToArray(typeof(string)));
		}

		private Hashtable GetRecordInfo(string searchformurl, string recordid)
		{
			string infourl = searchformurl.Replace("/search", "/info");
			HttpWebRequest req = (HttpWebRequest)HttpWebRequest.Create(infourl + "?" + recordid);
			WebResponse response = req.GetResponse();
			StreamReader reader = new StreamReader(response.GetResponseStream());
			string content = reader.ReadToEnd();
			reader.Close();

			Regex thumbnailregex = new Regex(
				"<td width=\"200\" align=\"center\" valign=\"top\">\\s*<a[^>]+><img[^>]+src=\"([^\"]+)\"", 
				RegexOptions.Singleline);
			string thumbnailurl = null;
			Match tm = thumbnailregex.Match(content);
			if (tm.Success)
				thumbnailurl = tm.Groups[1].Value;

			Hashtable result = new Hashtable();
			Regex regex = new Regex("<tr><th.+?scope=\"row\">([^<]+)</th>\\s*" +
				"<td>(.+?)</td></tr>", RegexOptions.Singleline);
			Regex removehtmlregex = new Regex("<.*?>", RegexOptions.Singleline);
			Regex collapsewhitespaceregex = new Regex("\\s+", RegexOptions.Singleline);
			Regex imageurlregex = new Regex("<a href=\"([^\"]+)\"(?!<a)", RegexOptions.Singleline);
			foreach (Match m in regex.Matches(content))
			{
				string field = HttpUtility.HtmlDecode(removehtmlregex.Replace(m.Groups[1].Value, ""));
				string val = HttpUtility.HtmlDecode(removehtmlregex.Replace(m.Groups[2].Value, ""));
				field = field.Replace(":", "");
				field = collapsewhitespaceregex.Replace(field, " ").Trim();
				val = collapsewhitespaceregex.Replace(val, " ").Trim();
				result.Add(field, val);
				if (field == "Format")
				{
					ArrayList a = new ArrayList();
					if (thumbnailurl != null)
						a.Add(thumbnailurl);
					foreach (Match mm in imageurlregex.Matches(m.Groups[2].Value))
						a.Add(mm.Groups[1].Value);
					result.Add("imageurl", a);
				}					
			}
			
			return result;
		}
	}
}
