Normalize forum subject/body HTML storage and migrate legacy encoded content

Dnn.CommunityForums/CustomControls/UserControls/TopicView.cs

@@ -313,9 +313,9 @@ private void LoadData(int pageId)
                     ModuleId = this.ForumModuleId,
                     ContentId = Utilities.SafeConvertInt(this.drForum["ContentId"]),
                     IPAddress = this.drForum["IPAddress"].ToString(),
-                    Subject = System.Net.WebUtility.HtmlDecode(this.drForum["Subject"].ToString()),
-                    Summary = System.Net.WebUtility.HtmlDecode(this.drForum["Summary"].ToString()),
-                    Body = System.Net.WebUtility.HtmlDecode(this.drForum["Body"].ToString()),
+                    Subject = this.drForum["Subject"].ToString(),
+                    Summary = this.drForum["Summary"].ToString(),
+                    Body = this.drForum["Body"].ToString(),
                     AuthorId = Utilities.SafeConvertInt(this.drForum["AuthorId"]),
                     AuthorName = this.drForum["TopicAuthor"].ToString(),
                     DateCreated = Utilities.SafeConvertDateTime(this.drForum["DateCreated"]),
@@ -1100,8 +1100,8 @@ private string ParseContent(DataRow dr, string template, int rowcount)
                     {
                         ModuleId = this.ForumModuleId,
                         ContentId = dr.GetInt("ContentId"),
-                        Body = System.Net.WebUtility.HtmlDecode(dr.GetString("Body")),
-                        Subject = System.Net.WebUtility.HtmlDecode(dr.GetString("Subject")),
+                        Body = dr.GetString("Body"),
+                        Subject = dr.GetString("Subject"),
                         AuthorId = dr.GetInt("AuthorId"),
                         DateCreated = dr.GetDateTime("DateCreated"),
                         DateUpdated = dr.GetDateTime("DateUpdated"),

Dnn.CommunityForums/CustomControls/UserControls/TopicsView.cs

@@ -478,10 +478,10 @@ private string ParseTopics(string Template, DataTable Topics, string Section)
                     {
                         ContentId = Convert.ToInt32(drTopic["TopicContentId"]),
                         ModuleId = this.ForumModuleId,
-                        Subject = System.Net.WebUtility.HtmlDecode(Convert.ToString(drTopic["Subject"])),
-                        Summary = System.Net.WebUtility.HtmlDecode(Convert.ToString(drTopic["Summary"])),
+                        Subject = Convert.ToString(drTopic["Subject"]),
+                        Summary = Convert.ToString(drTopic["Summary"]),
                         DateCreated = Convert.ToDateTime(drTopic["DateCreated"]),
-                        Body = System.Net.WebUtility.HtmlDecode(Convert.ToString(drTopic["Body"])),
+                        Body = Convert.ToString(drTopic["Body"]),
                         AuthorId = Convert.ToInt32(drTopic["AuthorId"]),
                         AuthorName = Convert.ToString(drTopic["AuthorName"]).ToString().Replace("&#", "&#"),
                     },
@@ -515,8 +515,8 @@ private string ParseTopics(string Template, DataTable Topics, string Section)
                         {
                             ContentId = Convert.ToInt32(drTopic["LastReplyContentId"]),
                             ModuleId = this.ForumModuleId,
-                            Subject = System.Net.WebUtility.HtmlDecode(Convert.ToString(drTopic["LastReplySubject"])),
-                            Summary = System.Net.WebUtility.HtmlDecode(Convert.ToString(drTopic["LastReplySummary"])),
+                            Subject = Convert.ToString(drTopic["LastReplySubject"]),
+                            Summary = Convert.ToString(drTopic["LastReplySummary"]),
                             DateCreated = Convert.ToDateTime(drTopic["LastReplyDate"]),
                             AuthorId = Convert.ToInt32(drTopic["LastReplyAuthorId"]),
                             AuthorName = Convert.ToString(drTopic["LastReplyAuthorName"]).ToString().Replace("&#", "&#"),

Dnn.CommunityForums/DnnCommunityForums.dnn

@@ -484,6 +484,11 @@
                 <name>09.07.00.SqlDataProvider</name>
                 <version>09.07.00</version>
             </script>
+            <script type="Install">
+                <path>sql</path>
+                <name>09.07.00-1607.SqlDataProvider</name>
+                <version>09.07.00</version>
+            </script>
             <script type="UnInstall">
               <path>sql</path>
               <name>Uninstall.SqlDataProvider</name>

Dnn.CommunityForums/Providers/DataProviders/SqlDataProvider/SqlDataProvider.cs

@@ -327,6 +327,8 @@ public override IDataReader Reply_Get(int PortalId, int ModuleId, int TopicId, i
 
         public override int Reply_Save(int PortalId, int TopicId, int ReplyId, int ReplyToId, int StatusId, bool IsApproved, bool IsDeleted, string Subject, string Body, DateTime DateCreated, DateTime DateUpdated, int AuthorId, string AuthorName, string IPAddress)
         {
+            Subject = Utilities.NormalizeHtmlForStorage(Subject);
+            Body = Utilities.NormalizeHtmlForStorage(Body);
             return Convert.ToInt32(SqlHelper.ExecuteScalar(this.ConnectionString, this.DatabaseOwner + this.ObjectQualifier + "activeforums_Reply_Save", PortalId, TopicId, ReplyId, ReplyToId, StatusId, IsApproved, IsDeleted, Subject, Body, DateCreated, DateUpdated, AuthorId, AuthorName, IPAddress));
         }
 
@@ -602,6 +604,9 @@ public override int Topics_Save(int PortalId, int TopicId, int ViewCount, int Re
 
         public override int Topics_Save(int PortalId, int ModuleId, int TopicId, int ViewCount, int ReplyCount, bool IsLocked, bool IsPinned, string TopicIcon, int StatusId, bool IsApproved, bool IsDeleted, bool IsAnnounce, bool IsArchived, DateTime AnnounceStart, DateTime AnnounceEnd, string Subject, string Body, string Summary, DateTime DateCreated, DateTime DateUpdated, int AuthorId, string AuthorName, string IPAddress, int TopicType, int priority, string URL, string TopicData)
         {
+            Subject = Utilities.NormalizeHtmlForStorage(Subject);
+            Body = Utilities.NormalizeHtmlForStorage(Body);
+            Summary = Utilities.NormalizeHtmlForStorage(Summary);
             return Convert.ToInt32(SqlHelper.ExecuteScalar(this.ConnectionString, this.DatabaseOwner + this.ObjectQualifier + "activeforums_Topics_Save", PortalId, ModuleId, TopicId, ViewCount, ReplyCount, IsLocked, IsPinned, TopicIcon, StatusId, IsApproved, IsDeleted, IsAnnounce, IsArchived, this.GetNull(AnnounceStart), this.GetNull(AnnounceEnd), Subject, Body, Summary, DateCreated, DateUpdated, AuthorId, AuthorName, IPAddress, TopicType, priority, URL, TopicData));
         }
 

Dnn.CommunityForums/Services/Tokens/TokenReplacer.cs

@@ -466,7 +466,7 @@ private static string ReplaceBody(string body, DotNetNuke.Modules.ActiveForums.M
         {
             if (!string.IsNullOrEmpty(body))
             {
-                body = Utilities.ManageImagePath(System.Net.WebUtility.HtmlDecode(body), uri);
+                body = Utilities.ManageImagePath(body, uri);
                 body = body.Replace("[", "[").Replace("]", "]");
                 if (body.ToUpper().Contains("[CODE]"))
                 {
@@ -476,7 +476,7 @@ private static string ReplaceBody(string body, DotNetNuke.Modules.ActiveForums.M
 
                 if (RegexUtils.GetCachedRegex("\\[CODE([^>]*)\\]", RegexOptions.Compiled & RegexOptions.IgnoreCase, 2).IsMatch(body))
                 {
-                    body = CodeParser.ParseCode(System.Net.WebUtility.HtmlDecode(body));
+                    body = CodeParser.ParseCode(body);
                 }
 
                 body = Utilities.StripExecCode(body);

Dnn.CommunityForums/class/Utilities.cs

@@ -26,6 +26,7 @@ namespace DotNetNuke.Modules.ActiveForums
     using System.Globalization;
     using System.IO;
     using System.Linq;
+    using System.Net;
     using System.Reflection;
     using System.Security.Cryptography;
     using System.Security.Principal;
@@ -546,6 +547,11 @@ internal static string DecodeBrackets(string text)
             return text.Replace("[", "[").Replace("]", "]").Replace("{", "{").Replace("}", "}");
         }
 
+        internal static string NormalizeHtmlForStorage(string text)
+        {
+            return string.IsNullOrEmpty(text) ? string.Empty : WebUtility.HtmlDecode(text);
+        }
+
         private static string CleanEditor(int portalId, string text, bool useFilter, int moduleId, string themePath, bool processEmoticons)
         {
             var strMessage = text;

Dnn.CommunityForums/controls/af_post.ascx.cs

@@ -371,12 +371,12 @@ private void LoadTopic()
             else
             {
                 // User has acccess
-                var sBody = System.Net.WebUtility.HtmlDecode(ti.Content.Body);
-                var sSubject = System.Net.WebUtility.HtmlDecode(ti.Content.Subject);
+                var sBody = ti.Content.Body;
+                var sSubject = ti.Content.Subject;
                 sBody = Utilities.PrepareForEdit(this.PortalId, this.ForumModuleId, this.ImagePath, sBody, this.allowHTML, this.editorType);
                 sSubject = Utilities.PrepareForEdit(this.PortalId, this.ForumModuleId, this.ImagePath, sSubject, false, EditorType.TEXTBOX);
                 this.ctlForm.Subject = sSubject;
-                this.ctlForm.Summary = System.Net.WebUtility.HtmlDecode(ti.Content.Summary);
+                this.ctlForm.Summary = ti.Content.Summary;
                 this.ctlForm.Body = sBody;
                 this.ctlForm.AnnounceEnd = ti.AnnounceEnd ?? Utilities.NullDate();
                 this.ctlForm.AnnounceStart = ti.AnnounceStart ?? Utilities.NullDate();
@@ -456,8 +456,8 @@ private void LoadReply()
             }
             else
             {
-                var sBody = System.Net.WebUtility.HtmlDecode(ri.Content.Body);
-                var sSubject = System.Net.WebUtility.HtmlDecode(ri.Content.Subject);
+                var sBody = ri.Content.Body;
+                var sSubject = ri.Content.Subject;
                 sBody = Utilities.PrepareForEdit(this.PortalId, this.ForumModuleId, this.ImagePath, sBody, this.allowHTML, this.editorType);
                 sSubject = Utilities.PrepareForEdit(this.PortalId, this.ForumModuleId, this.ImagePath, sSubject, false, EditorType.TEXTBOX);
                 this.ctlForm.Subject = sSubject;
@@ -554,8 +554,8 @@ private void PrepareReply()
                 }
                 else
                 {
-                    this.ctlForm.Subject = Utilities.GetSharedResource("[RESX:SubjectPrefix]") + " " + System.Net.WebUtility.HtmlDecode(ti.Content.Subject);
-                    this.ctlForm.TopicSubject = System.Net.WebUtility.HtmlDecode(ti.Content.Subject);
+                    this.ctlForm.Subject = Utilities.GetSharedResource("[RESX:SubjectPrefix]") + " " + ti.Content.Subject;
+                    this.ctlForm.TopicSubject = ti.Content.Subject;
                     if (ti.IsLocked && (this.ForumUser.CurrentUserType == CurrentUserTypes.Anon || this.ForumUser.CurrentUserType == CurrentUserTypes.Auth))
                     {
                         this.Response.Redirect(this.NavigateUrl(this.TabId), false);
@@ -657,7 +657,9 @@ private void SaveTopic()
             var body = this.ctlForm.Body;
             subject = Utilities.CleanString(this.PortalId, Utilities.XSSFilter(subject, true), false, EditorType.TEXTBOX, this.ForumInfo.FeatureSettings.UseFilter, false, this.ForumModuleId, this.themePath, false);
             body = Utilities.CleanString(this.PortalId, body, this.allowHTML, this.editorType, this.ForumInfo.FeatureSettings.UseFilter, this.ForumInfo.FeatureSettings.AllowScript, this.ForumModuleId, this.themePath, this.ForumInfo.FeatureSettings.AllowEmoticons);
-            var summary = this.ctlForm.Summary;
+            subject = Utilities.NormalizeHtmlForStorage(subject);
+            body = Utilities.NormalizeHtmlForStorage(body);
+            var summary = Utilities.NormalizeHtmlForStorage(this.ctlForm.Summary);
             int authorId;
             string authorName;
             if (this.Request.IsAuthenticated)
@@ -921,6 +923,8 @@ private void SaveReply()
             var body = this.ctlForm.Body;
             subject = Utilities.CleanString(this.PortalId, subject, false, EditorType.TEXTBOX, this.ForumInfo.FeatureSettings.UseFilter, false, this.ForumModuleId, this.themePath, false);
             body = Utilities.CleanString(this.PortalId, body, this.allowHTML, this.editorType, this.ForumInfo.FeatureSettings.UseFilter, this.ForumInfo.FeatureSettings.AllowScript, this.ForumModuleId, this.themePath, this.ForumInfo.FeatureSettings.AllowEmoticons);
+            subject = Utilities.NormalizeHtmlForStorage(subject);
+            body = Utilities.NormalizeHtmlForStorage(body);
 
             // This HTML decode is used to make Quote functionality work properly even when it appears in Text Box instead of Editor
             if (this.Request.Params[ParamKeys.QuoteId] != null)

Dnn.CommunityForums/sql/09.07.00-1607.SqlDataProvider

@@ -0,0 +1,23 @@
+SET NOCOUNT ON
+GO
+
+/* issue 1607 - begin - html entity normalization for activeforums_Content */
+
+/*
+Most recent procedure versions that persist topic/reply content:
+- activeforums_Topics_Save      : 09.01.00.SqlDataProvider
+- activeforums_Reply_Save       : 09.02.00.SqlDataProvider
+- activeforums_Topics_SaveToForum: 08.02.00.SqlDataProvider
+*/
+
+/* decode order matters: double-encoded entities first, then single-encoded entities, and '&' last */
+/* this is a one-time module-upgrade data fix for legacy content */
+UPDATE {databaseOwner}[{objectQualifier}activeforums_Content]
+SET Subject = REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(Subject, '&amp;lt;', '&lt;'), '&amp;gt;', '&gt;'), '&amp;quot;', '&quot;'), '&amp;#39;', '&#39;'), '&amp;#91;', '&#91;'), '&amp;#93;', '&#93;'), '&amp;amp;', '&amp;'), '&lt;', '<'), '&gt;', '>'), '&quot;', '"'), '&#39;', ''''), '&amp;', '&'),
+    Body = REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(REPLACE(Body, '&amp;lt;', '&lt;'), '&amp;gt;', '&gt;'), '&amp;quot;', '&quot;'), '&amp;#39;', '&#39;'), '&amp;#91;', '&#91;'), '&amp;#93;', '&#93;'), '&amp;amp;', '&amp;'), '&lt;', '<'), '&gt;', '>'), '&quot;', '"'), '&#39;', ''''), '&amp;', '&'),
+    DateUpdated = GETUTCDATE()
+WHERE Subject LIKE '%&amp;%' OR Subject LIKE '%&lt;%' OR Subject LIKE '%&gt;%' OR Subject LIKE '%&#39;%' OR Subject LIKE '%&quot;%'
+   OR Body LIKE '%&amp;%' OR Body LIKE '%&lt;%' OR Body LIKE '%&gt;%' OR Body LIKE '%&#39;%' OR Body LIKE '%&quot;%';
+GO
+
+/* issue 1607 - end - html entity normalization for activeforums_Content */

Dnn.CommunityForumsTests/class/UtilitiesTests.cs

@@ -447,6 +447,29 @@ public void DecodeBrackets_WithPartialEncodedBrackets_OnlyDecodesComplete()
             Assert.That(result, Is.EqualTo(expected));
         }
 
+        [Test]
+        [TestCase("A & B", "A & B")]
+        [TestCase("<div>test</div>", "<div>test</div>")]
+        [TestCase("", "")]
+        public void NormalizeHtmlForStorage_DecodesHtmlOnce(string input, string expected)
+        {
+            // Act
+            var result = Utilities.NormalizeHtmlForStorage(input);
+
+            // Assert
+            Assert.That(result, Is.EqualTo(expected));
+        }
+
+        [Test]
+        public void NormalizeHtmlForStorage_WithNull_ReturnsEmptyString()
+        {
+            // Act
+            var result = Utilities.NormalizeHtmlForStorage(null);
+
+            // Assert
+            Assert.That(result, Is.EqualTo(string.Empty));
+        }
+
         [Test]
         public void GetSha256HashTest()
         {