From 8fd59d6f336a95782a13b0440a6cf5108e58a1f8 Mon Sep 17 00:00:00 2001 From: audrey-inglish <112636119+audrey-inglish@users.noreply.github.com> Date: Mon, 8 Dec 2025 10:43:37 -0700 Subject: [PATCH] Merge pull request #14879 from audrey-inglish/master Fix: normalize punctuation when computing CleanName so searches without punctuation match (closes #1674) --- .../Item/BaseItemRepository.cs | 44 ++++++- .../Migrations/Routines/RefreshCleanNames.cs | 105 +++++++++++++++++ .../Data/SearchPunctuationTests.cs | 109 ++++++++++++++++++ 3 files changed, 256 insertions(+), 2 deletions(-) create mode 100644 Jellyfin.Server/Migrations/Routines/RefreshCleanNames.cs create mode 100644 tests/Jellyfin.Server.Implementations.Tests/Data/SearchPunctuationTests.cs diff --git a/Jellyfin.Server.Implementations/Item/BaseItemRepository.cs b/Jellyfin.Server.Implementations/Item/BaseItemRepository.cs index 84168291a8..57d874e59f 100644 --- a/Jellyfin.Server.Implementations/Item/BaseItemRepository.cs +++ b/Jellyfin.Server.Implementations/Item/BaseItemRepository.cs @@ -1373,14 +1373,54 @@ public sealed class BaseItemRepository } } - private string GetCleanValue(string value) + /// + /// Gets the clean value for search and sorting purposes. + /// + /// The value to clean. + /// The cleaned value. + public static string GetCleanValue(string value) { if (string.IsNullOrWhiteSpace(value)) { return value; } - return value.RemoveDiacritics().ToLowerInvariant(); + var noDiacritics = value.RemoveDiacritics(); + + // Build a string where any punctuation or symbol is treated as a separator (space). + var sb = new StringBuilder(noDiacritics.Length); + var previousWasSpace = false; + foreach (var ch in noDiacritics) + { + char outCh; + if (char.IsLetterOrDigit(ch) || char.IsWhiteSpace(ch)) + { + outCh = ch; + } + else + { + outCh = ' '; + } + + // normalize any whitespace character to a single ASCII space. + if (char.IsWhiteSpace(outCh)) + { + if (!previousWasSpace) + { + sb.Append(' '); + previousWasSpace = true; + } + } + else + { + sb.Append(outCh); + previousWasSpace = false; + } + } + + // trim leading/trailing spaces that may have been added. + var collapsed = sb.ToString().Trim(); + return collapsed.ToLowerInvariant(); } private List<(ItemValueType MagicNumber, string Value)> GetItemValuesToSave(BaseItemDto item, List inheritedTags) diff --git a/Jellyfin.Server/Migrations/Routines/RefreshCleanNames.cs b/Jellyfin.Server/Migrations/Routines/RefreshCleanNames.cs new file mode 100644 index 0000000000..eadabf6776 --- /dev/null +++ b/Jellyfin.Server/Migrations/Routines/RefreshCleanNames.cs @@ -0,0 +1,105 @@ +using System; +using System.Diagnostics; +using System.Linq; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Jellyfin.Database.Implementations; +using Jellyfin.Database.Implementations.Entities; +using Jellyfin.Extensions; +using Jellyfin.Server.Implementations.Item; +using Jellyfin.Server.ServerSetupApp; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Logging; + +namespace Jellyfin.Server.Migrations.Routines; + +/// +/// Migration to refresh CleanName values for all library items. +/// +[JellyfinMigration("2025-10-08T12:00:00", nameof(RefreshCleanNames))] +[JellyfinMigrationBackup(JellyfinDb = true)] +public class RefreshCleanNames : IAsyncMigrationRoutine +{ + private readonly IStartupLogger _logger; + private readonly IDbContextFactory _dbProvider; + + /// + /// Initializes a new instance of the class. + /// + /// The logger. + /// Instance of the interface. + public RefreshCleanNames( + IStartupLogger logger, + IDbContextFactory dbProvider) + { + _logger = logger; + _dbProvider = dbProvider; + } + + /// + public async Task PerformAsync(CancellationToken cancellationToken) + { + const int Limit = 1000; + int itemCount = 0; + + var sw = Stopwatch.StartNew(); + + using var context = _dbProvider.CreateDbContext(); + var records = context.BaseItems.Count(b => !string.IsNullOrEmpty(b.Name)); + _logger.LogInformation("Refreshing CleanName for {Count} library items", records); + + var processedInPartition = 0; + + await foreach (var item in context.BaseItems + .Where(b => !string.IsNullOrEmpty(b.Name)) + .OrderBy(e => e.Id) + .WithPartitionProgress((partition) => _logger.LogInformation("Processed: {Offset}/{Total} - Updated: {UpdatedCount} - Time: {Elapsed}", partition * Limit, records, itemCount, sw.Elapsed)) + .PartitionEagerAsync(Limit, cancellationToken) + .WithCancellation(cancellationToken) + .ConfigureAwait(false)) + { + try + { + var newCleanName = string.IsNullOrWhiteSpace(item.Name) ? string.Empty : BaseItemRepository.GetCleanValue(item.Name); + if (!string.Equals(newCleanName, item.CleanName, StringComparison.Ordinal)) + { + _logger.LogDebug( + "Updating CleanName for item {Id}: '{OldValue}' -> '{NewValue}'", + item.Id, + item.CleanName, + newCleanName); + item.CleanName = newCleanName; + itemCount++; + } + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Failed to update CleanName for item {Id} ({Name})", item.Id, item.Name); + } + + processedInPartition++; + + if (processedInPartition >= Limit) + { + await context.SaveChangesAsync(cancellationToken).ConfigureAwait(false); + // Clear tracked entities to avoid memory growth across partitions + context.ChangeTracker.Clear(); + processedInPartition = 0; + } + } + + // Save any remaining changes after the loop + if (processedInPartition > 0) + { + await context.SaveChangesAsync(cancellationToken).ConfigureAwait(false); + context.ChangeTracker.Clear(); + } + + _logger.LogInformation( + "Refreshed CleanName for {UpdatedCount} out of {TotalCount} items in {Time}", + itemCount, + records, + sw.Elapsed); + } +} diff --git a/tests/Jellyfin.Server.Implementations.Tests/Data/SearchPunctuationTests.cs b/tests/Jellyfin.Server.Implementations.Tests/Data/SearchPunctuationTests.cs new file mode 100644 index 0000000000..8fbccd8019 --- /dev/null +++ b/tests/Jellyfin.Server.Implementations.Tests/Data/SearchPunctuationTests.cs @@ -0,0 +1,109 @@ +using System; +using AutoFixture; +using AutoFixture.AutoMoq; +using Jellyfin.Server.Implementations.Item; +using MediaBrowser.Controller.Entities.TV; +using Microsoft.Extensions.Configuration; +using Moq; +using Xunit; + +namespace Jellyfin.Server.Implementations.Tests.Data +{ + public class SearchPunctuationTests + { + private readonly IFixture _fixture; + private readonly BaseItemRepository _repo; + + public SearchPunctuationTests() + { + var appHost = new Mock(); + appHost.Setup(x => x.ExpandVirtualPath(It.IsAny())) + .Returns((string x) => x); + appHost.Setup(x => x.ReverseVirtualPath(It.IsAny())) + .Returns((string x) => x); + + var configSection = new Mock(); + configSection + .SetupGet(x => x[It.Is(s => s == MediaBrowser.Controller.Extensions.ConfigurationExtensions.SqliteCacheSizeKey)]) + .Returns("0"); + var config = new Mock(); + config + .Setup(x => x.GetSection(It.Is(s => s == MediaBrowser.Controller.Extensions.ConfigurationExtensions.SqliteCacheSizeKey))) + .Returns(configSection.Object); + + _fixture = new Fixture().Customize(new AutoMoqCustomization { ConfigureMembers = true }); + _fixture.Inject(appHost.Object); + _fixture.Inject(config.Object); + + _repo = _fixture.Create(); + } + + [Fact] + public void CleanName_keeps_punctuation_and_search_without_punctuation_passes() + { + var series = new Series + { + Id = Guid.NewGuid(), + Name = "Mr. Robot" + }; + + series.SortName = "Mr. Robot"; + + var entity = _repo.Map(series); + Assert.Equal("mr robot", entity.CleanName); + + var searchTerm = "Mr Robot".ToLowerInvariant(); + + Assert.Contains(searchTerm, entity.CleanName ?? string.Empty, StringComparison.OrdinalIgnoreCase); + } + + [Theory] + [InlineData("Spider-Man: Homecoming", "spider man homecoming")] + [InlineData("Beyoncé — Live!", "beyonce live")] + [InlineData("Hello, World!", "hello world")] + [InlineData("(The) Good, the Bad & the Ugly", "the good the bad the ugly")] + [InlineData("Wall-E", "wall e")] + [InlineData("No. 1: The Beginning", "no 1 the beginning")] + [InlineData("Café-au-lait", "cafe au lait")] + public void CleanName_normalizes_various_punctuation(string title, string expectedClean) + { + var series = new Series + { + Id = Guid.NewGuid(), + Name = title + }; + + series.SortName = title; + + var entity = _repo.Map(series); + + Assert.Equal(expectedClean, entity.CleanName); + + // Ensure a search term without punctuation would match + var searchTerm = expectedClean; + Assert.Contains(searchTerm, entity.CleanName ?? string.Empty, StringComparison.OrdinalIgnoreCase); + } + + [Theory] + [InlineData("Face/Off", "face off")] + [InlineData("V/H/S", "v h s")] + public void CleanName_normalizes_titles_withslashes(string title, string expectedClean) + { + var series = new Series + { + Id = Guid.NewGuid(), + Name = title + }; + + series.SortName = title; + + var entity = _repo.Map(series); + + Assert.Equal(expectedClean, entity.CleanName); + + // Ensure a search term without punctuation would match + var searchTerm = expectedClean; + Assert.Contains(searchTerm, entity.CleanName ?? string.Empty, StringComparison.OrdinalIgnoreCase); + } + } +}