Added "Page Not Found" check for LyricsFreak provider. Fixed searching for some cases. Added Integration test.

Author: skuill

LyricsScraperNET/Providers/LyricsFreak/LyricsFreakProvider.cs

@@ -16,9 +16,12 @@ internal class LyricsFreakProvider : ExternalProviderBase
     {
         private ILogger<LyricsFreakProvider>? _logger;
         private readonly IExternalUriConverter _uriConverter;
-        private readonly string LyricsHrefXPath = "//a[translate(@title, 'ABCDEFGHIJKLMNOPQRSTUVWXYZ', 'abcdefghijklmnopqrstuvwxyz') = '{0} lyrics']";
+        
+        private const string LyricsHrefXPath = "//a[contains(translate(@title, 'ABCDEFGHIJKLMNOPQRSTUVWXYZ', 'abcdefghijklmnopqrstuvwxyz'), '{0} lyrics')]";
         private const string LyricsDivXPath = "//div[@data-container-id='lyrics']";
 
+        private const string PageNotFoundText = "#404 - Page Not Found";
+
         public override IExternalProviderOptions Options { get; }
 
         #region Constructors
@@ -90,11 +93,17 @@ protected override async Task<SearchResult> SearchLyricAsync(string artist, stri
 
             cancellationToken.ThrowIfCancellationRequested();
 
+            if (htmlResponse.Contains(PageNotFoundText))
+            {
+                _logger?.LogWarning($"LyricsFreak. Artist's page not found (404). [{artist}]. Song name: [{song}]");
+                return new SearchResult(Models.ExternalProviderType.LyricsFreak);
+            }
+
             // 2. Find song on the artist page and get link to the web page.
             var songHref = GetSongHrefFromHtmlBody(htmlResponse, song);
             if (string.IsNullOrEmpty(songHref))
             {
-                _logger?.LogWarning($"LyricsFreak. Can't find song Uri for song: [{song}]");
+                _logger?.LogWarning($"LyricsFreak. Can't find song Uri for artist: [{artist}]. Song name: [{song}]");
                 return new SearchResult(Models.ExternalProviderType.LyricsFreak);
             }
             var songUri = new Uri(LyricsFreakUriConverter.BaseUrl + songHref);
@@ -104,14 +113,14 @@ protected override async Task<SearchResult> SearchLyricAsync(string artist, stri
 
         protected async override Task<SearchResult> SearchLyricAsync(Uri uri, CancellationToken cancellationToken = default)
         {
-            var text = await WebClient.LoadAsync(uri, cancellationToken);
+            var htmlBodyContent = await WebClient.LoadAsync(uri, cancellationToken);
 
             cancellationToken.ThrowIfCancellationRequested();
 
-            var songLyrics = GetSongLyricsFromHtmlBody(text);
+            var songLyrics = GetSongLyricsFromHtmlBody(htmlBodyContent);
             if (string.IsNullOrEmpty(songLyrics))
             {
-                _logger?.LogWarning($"LyricsFreak. Can't find song lyrics for song uri: [{uri.AbsoluteUri}]");
+                _logger?.LogWarning($"LyricsFreak. Can't find lyrics for song's uri: [{uri}]");
                 return new SearchResult(Models.ExternalProviderType.LyricsFreak);
             }
 
@@ -126,12 +135,20 @@ protected async override Task<SearchResult> SearchLyricAsync(Uri uri, Cancellati
 
         private string GetSongHrefFromHtmlBody(string htmlBody, string song)
         {
+            // Encoded needed for songs like "Devil's Calling". Title in htmlBody will be: "Devil&#039;s Calling Lyrics"
             string formattedXPath = string.Format(LyricsHrefXPath, GetEncodedSong(song));
-            var linkNode = htmlBody.SelectSingleNodeByXPath(formattedXPath);
+
+            // In other cases tried lowercase search of the original song name. 
+            // Example. Artist: "Zé Ramalho". Song: "Batendo Na Porta Do Céu (Versão II)"
+            string originalXPath = string.Format(LyricsHrefXPath, song.ToLowerInvariant());
+
+            var linkNode = htmlBody.SelectSingleNodeByXPath(formattedXPath)
+                ?? (!song.Contains("'") 
+                    ? htmlBody.SelectSingleNodeByXPath(originalXPath) 
+                    : null);
+
             if (linkNode == null)
-            {
                 return string.Empty;
-            }
 
             string hrefSong = linkNode.GetAttributeValue("href", string.Empty);
             return hrefSong;
@@ -140,11 +157,10 @@ private string GetSongHrefFromHtmlBody(string htmlBody, string song)
         private string GetSongLyricsFromHtmlBody(string htmlBody)
         {
             var lyricsNode = htmlBody.SelectSingleNodeByXPath(LyricsDivXPath);
+
             if (lyricsNode == null)
-            {
                 return string.Empty;
 
-            }
             string lyricsText = lyricsNode.InnerText.Trim();
             return lyricsText;
         }

LyricsScraperNET/Providers/LyricsFreak/LyricsFreakUriConverter.cs

@@ -17,7 +17,6 @@ public Uri GetArtistUri(string artist)
         {
             var artistFormatted = artist.ToLowerInvariant().СonvertToPlusFormat(removeProhibitedSymbols: true);
             return new Uri(string.Format(uriArtistPathFormat, artistFormatted.First(c => c != '+'), artistFormatted));
-
         }
 
         public Uri GetLyricUri(string artist, string song)

Tests/LyricsScraperNET.IntegrationTest/LyricsScraperNET.IntegrationTest.csproj

@@ -56,6 +56,9 @@
 		<None Update="Providers\LyricsFreak\Resources\Lyrics_Result_01.txt">
 		  <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
 		</None>
+		<None Update="Providers\LyricsFreak\Resources\Lyrics_Result_03.txt">
+		  <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+		</None>
 		<None Update="Providers\LyricsFreak\Resources\Lyrics_Result_02.txt">
 		  <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
 		</None>

Tests/LyricsScraperNET.IntegrationTest/Providers/LyricsFreak/Resources/Lyrics_Result_03.txt

@@ -0,0 +1,15 @@
+Mãe, tire essas algemas de mim
+Me proteja com o seu véyou
+Está escuro demais pra ver
+Me sinto até batendo na porta do céyou
+
+Bate, Bate, Bate na porta do céyou
+Bate, Bate, Bate na porta do céyou
+
+Mãe, tire essas armas pra mim
+A camisa e o chapéyou
+A grande nuvem escura já me envolveu
+Me sinto até batendo na porta do céyou
+
+Bate, bate, bate, bate na porta do céyou
+Bate, bate, bate na porta do céyou

Tests/LyricsScraperNET.IntegrationTest/Providers/LyricsFreak/lyric_test_data.json

@@ -11,6 +11,12 @@
     "SongName": "Wishing Wells",
     "SongUri": null
   },
+  {
+    "LyricResultPath": "Providers/LyricsFreak/Resources/Lyrics_Result_03.txt",
+    "ArtistName": "Zé Ramalho",
+    "SongName": "Batendo Na Porta Do Céu (Versão II)",
+    "SongUri": null
+  },
   {
     "LyricResultPath": "Providers/LyricsFreak/Resources/Lyrics_Result_01.txt",
     "ArtistName": null,