chore: Change PDF text extraction logics (#10420)

* chore: change PDF text extraction logics

* test(snapshot): update snapshots c2df7e0

* test(snapshot): update snapshots a124734

---------

Co-authored-by: filzrev <[email protected]>
Co-authored-by: Yufei Huang <[email protected]>

test/docfx.Snapshot.Tests/SamplesTest.cs

@@ -11,6 +11,8 @@
 using UglyToad.PdfPig;
 using UglyToad.PdfPig.Actions;
 using UglyToad.PdfPig.Annotations;
+using UglyToad.PdfPig.Content;
+using UglyToad.PdfPig.DocumentLayoutAnalysis.TextExtractor;
 using UglyToad.PdfPig.Outline;
 
 namespace Docfx.Tests;
@@ -80,7 +82,7 @@ void PdfToJson(string path)
                 {
                     p.Number,
                     p.NumberOfImages,
-                    p.Text,
+                    Text = ExtractText(p),
                     Links = p.ExperimentalAccess.GetAnnotations().Select(ToLink).ToArray(),
                 }).ToArray(),
                 Bookmarks = document.TryGetBookmarks(out var bookmarks) ? ToBookmarks(bookmarks.Roots) : null,
@@ -207,4 +209,24 @@ private void ScrubFile(string path, StringBuilder builder)
             }));
         }
     }
+
+    private string ExtractText(Page page)
+    {
+        // Gets PDF text content
+        var text = ContentOrderTextExtractor.GetText(page, new ContentOrderTextExtractor.Options { ReplaceWhitespaceWithSpace = true });
+
+        // string.Normalize is not works when using `Globalization Invariant Mode`.
+        StringBuilder sb = new(text);
+
+        // Normalize known ligature chars. (Note: `string.Normalize` is not works when using `Globalization Invariant Mode`)
+        sb.Replace("ff", "ff");
+        sb.Replace("ffi", "ffi");
+        sb.Replace("fl", "fl");
+        sb.Replace("fi", "fi");
+
+        // Normalize newline char.
+        sb.Replace("\r\n", "\n");
+
+        return sb.ToString();
+    }
 }