Backported Metal sampler improvements from main

Fixes https://github.com/libsdl-org/SDL/issues/12988

src/render/metal/SDL_render_metal.m

@@ -79,15 +79,10 @@ static const size_t CONSTANTS_OFFSET_DECODE_BT2020_LIMITED = ALIGN_CONSTANTS(16,
 static const size_t CONSTANTS_OFFSET_DECODE_BT2020_FULL = ALIGN_CONSTANTS(16, CONSTANTS_OFFSET_DECODE_BT2020_LIMITED + sizeof(float) * 4 * 4);
 static const size_t CONSTANTS_LENGTH = CONSTANTS_OFFSET_DECODE_BT2020_FULL + sizeof(float) * 4 * 4;
 
-// Sampler types
-typedef enum
-{
-    SDL_METAL_SAMPLER_NEAREST_CLAMP,
-    SDL_METAL_SAMPLER_NEAREST_WRAP,
-    SDL_METAL_SAMPLER_LINEAR_CLAMP,
-    SDL_METAL_SAMPLER_LINEAR_WRAP,
-    SDL_NUM_METAL_SAMPLERS
-} SDL_METAL_sampler_type;
+#define RENDER_SAMPLER_HASHKEY(scale_mode, address_u, address_v)    \
+    (((scale_mode == SDL_SCALEMODE_NEAREST) << 0) |                 \
+     ((address_u == SDL_TEXTURE_ADDRESS_WRAP) << 1) |               \
+     ((address_v == SDL_TEXTURE_ADDRESS_WRAP) << 2))
 
 typedef enum SDL_MetalVertexFunction
 {
@@ -139,7 +134,7 @@ typedef struct METAL_ShaderPipelines
 @property(nonatomic, retain) id<MTLRenderCommandEncoder> mtlcmdencoder;
 @property(nonatomic, retain) id<MTLLibrary> mtllibrary;
 @property(nonatomic, retain) id<CAMetalDrawable> mtlbackbuffer;
-@property(nonatomic, retain) NSMutableArray<id<MTLSamplerState>> *mtlsamplers;
+@property(nonatomic, retain) NSMutableDictionary<NSNumber *, id<MTLSamplerState>> *mtlsamplers;
 @property(nonatomic, retain) id<MTLBuffer> mtlbufconstants;
 @property(nonatomic, retain) id<MTLBuffer> mtlbufquadindices;
 @property(nonatomic, assign) SDL_MetalView mtlview;
@@ -1295,6 +1290,9 @@ typedef struct
     __unsafe_unretained id<MTLBuffer> vertex_buffer;
     size_t constants_offset;
     SDL_Texture *texture;
+    SDL_ScaleMode texture_scale_mode;
+    SDL_TextureAddressMode texture_address_mode_u;
+    SDL_TextureAddressMode texture_address_mode_v;
     bool cliprect_dirty;
     bool cliprect_enabled;
     SDL_Rect cliprect;
@@ -1452,6 +1450,58 @@ static bool SetDrawState(SDL_Renderer *renderer, const SDL_RenderCommand *cmd, c
     return true;
 }
 
+static id<MTLSamplerState> GetSampler(SDL3METAL_RenderData *data, SDL_ScaleMode scale_mode, SDL_TextureAddressMode address_u, SDL_TextureAddressMode address_v)
+{
+    NSNumber *key = [NSNumber numberWithInteger:RENDER_SAMPLER_HASHKEY(scale_mode, address_u, address_v)];
+    id<MTLSamplerState> mtlsampler = data.mtlsamplers[key];
+    if (mtlsampler == nil) {
+        MTLSamplerDescriptor *samplerdesc;
+        samplerdesc = [[MTLSamplerDescriptor alloc] init];
+        switch (scale_mode) {
+        case SDL_SCALEMODE_NEAREST:
+            samplerdesc.minFilter = MTLSamplerMinMagFilterNearest;
+            samplerdesc.magFilter = MTLSamplerMinMagFilterNearest;
+            break;
+        case SDL_SCALEMODE_LINEAR:
+            samplerdesc.minFilter = MTLSamplerMinMagFilterLinear;
+            samplerdesc.magFilter = MTLSamplerMinMagFilterLinear;
+            break;
+        default:
+            SDL_SetError("Unknown scale mode: %d", scale_mode);
+            return nil;
+        }
+        switch (address_u) {
+        case SDL_TEXTURE_ADDRESS_CLAMP:
+            samplerdesc.sAddressMode = MTLSamplerAddressModeClampToEdge;
+            break;
+        case SDL_TEXTURE_ADDRESS_WRAP:
+            samplerdesc.sAddressMode = MTLSamplerAddressModeRepeat;
+            break;
+        default:
+            SDL_SetError("Unknown texture address mode: %d", address_u);
+            return nil;
+        }
+        switch (address_v) {
+        case SDL_TEXTURE_ADDRESS_CLAMP:
+            samplerdesc.tAddressMode = MTLSamplerAddressModeClampToEdge;
+            break;
+        case SDL_TEXTURE_ADDRESS_WRAP:
+            samplerdesc.tAddressMode = MTLSamplerAddressModeRepeat;
+            break;
+        default:
+            SDL_SetError("Unknown texture address mode: %d", address_v);
+            return nil;
+        }
+        mtlsampler = [data.mtldevice newSamplerStateWithDescriptor:samplerdesc];
+        if (mtlsampler == nil) {
+            SDL_SetError("Couldn't create sampler");
+            return nil;
+        }
+        data.mtlsamplers[key] = mtlsampler;
+    }
+    return mtlsampler;
+}
+
 static bool SetCopyState(SDL_Renderer *renderer, const SDL_RenderCommand *cmd, const size_t constants_offset,
                              id<MTLBuffer> mtlbufvertex, METAL_DrawStateCache *statecache)
 {
@@ -1467,33 +1517,6 @@ static bool SetCopyState(SDL_Renderer *renderer, const SDL_RenderCommand *cmd, c
     }
 
     if (texture != statecache->texture) {
-        id<MTLSamplerState> mtlsampler;
-
-        if (cmd->data.draw.texture_scale_mode == SDL_SCALEMODE_NEAREST) {
-            switch (cmd->data.draw.texture_address_mode) {
-            case SDL_TEXTURE_ADDRESS_CLAMP:
-                mtlsampler = data.mtlsamplers[SDL_METAL_SAMPLER_NEAREST_CLAMP];
-                break;
-            case SDL_TEXTURE_ADDRESS_WRAP:
-                mtlsampler = data.mtlsamplers[SDL_METAL_SAMPLER_NEAREST_WRAP];
-                break;
-            default:
-                return SDL_SetError("Unknown texture address mode: %d", cmd->data.draw.texture_address_mode);
-            }
-        } else {
-            switch (cmd->data.draw.texture_address_mode) {
-            case SDL_TEXTURE_ADDRESS_CLAMP:
-                mtlsampler = data.mtlsamplers[SDL_METAL_SAMPLER_LINEAR_CLAMP];
-                break;
-            case SDL_TEXTURE_ADDRESS_WRAP:
-                mtlsampler = data.mtlsamplers[SDL_METAL_SAMPLER_LINEAR_WRAP];
-                break;
-            default:
-                return SDL_SetError("Unknown texture address mode: %d", cmd->data.draw.texture_address_mode);
-            }
-        }
-        [data.mtlcmdencoder setFragmentSamplerState:mtlsampler atIndex:0];
-
         [data.mtlcmdencoder setFragmentTexture:texturedata.mtltexture atIndex:0];
 #ifdef SDL_HAVE_YUV
         if (texturedata.yuv || texturedata.nv12) {
@@ -1503,6 +1526,20 @@ static bool SetCopyState(SDL_Renderer *renderer, const SDL_RenderCommand *cmd, c
 #endif
         statecache->texture = texture;
     }
+
+    if (cmd->data.draw.texture_scale_mode != statecache->texture_scale_mode ||
+        cmd->data.draw.texture_address_mode != statecache->texture_address_mode_u ||
+        cmd->data.draw.texture_address_mode != statecache->texture_address_mode_v) {
+        id<MTLSamplerState> mtlsampler = GetSampler(data, cmd->data.draw.texture_scale_mode, cmd->data.draw.texture_address_mode, cmd->data.draw.texture_address_mode);
+        if (mtlsampler == nil) {
+            return false;
+        }
+        [data.mtlcmdencoder setFragmentSamplerState:mtlsampler atIndex:0];
+
+        statecache->texture_scale_mode = cmd->data.draw.texture_scale_mode;
+        statecache->texture_address_mode_u = cmd->data.draw.texture_address_mode;
+        statecache->texture_address_mode_v = cmd->data.draw.texture_address_mode;
+    }
     return true;
 }
 
@@ -1523,6 +1560,9 @@ static bool METAL_RunCommandQueue(SDL_Renderer *renderer, SDL_RenderCommand *cmd
         statecache.vertex_buffer = nil;
         statecache.constants_offset = CONSTANTS_OFFSET_INVALID;
         statecache.texture = NULL;
+        statecache.texture_scale_mode = SDL_SCALEMODE_INVALID;
+        statecache.texture_address_mode_u = SDL_TEXTURE_ADDRESS_INVALID;
+        statecache.texture_address_mode_v = SDL_TEXTURE_ADDRESS_INVALID;
         statecache.shader_constants_dirty = true;
         statecache.cliprect_dirty = true;
         statecache.viewport_dirty = true;
@@ -1883,7 +1923,6 @@ static bool METAL_CreateRenderer(SDL_Renderer *renderer, SDL_Window *window, SDL
         int maxtexsize, quadcount = UINT16_MAX / 4;
         UInt16 *indexdata;
         size_t indicessize = sizeof(UInt16) * quadcount * 6;
-        MTLSamplerDescriptor *samplerdesc;
         id<MTLCommandQueue> mtlcmdqueue;
         id<MTLLibrary> mtllibrary;
         id<MTLBuffer> mtlbufconstantstaging, mtlbufquadindicesstaging, mtlbufconstants, mtlbufquadindices;
@@ -2043,27 +2082,7 @@ static bool METAL_CreateRenderer(SDL_Renderer *renderer, SDL_Window *window, SDL
         data.allpipelines = NULL;
         ChooseShaderPipelines(data, MTLPixelFormatBGRA8Unorm);
 
-        static struct
-        {
-            MTLSamplerMinMagFilter filter;
-            MTLSamplerAddressMode address;
-        } samplerParams[] = {
-            { MTLSamplerMinMagFilterNearest, MTLSamplerAddressModeClampToEdge },
-            { MTLSamplerMinMagFilterNearest, MTLSamplerAddressModeRepeat },
-            { MTLSamplerMinMagFilterLinear, MTLSamplerAddressModeClampToEdge },
-            { MTLSamplerMinMagFilterLinear, MTLSamplerAddressModeRepeat },
-        };
-        SDL_COMPILE_TIME_ASSERT(samplerParams_SIZE, SDL_arraysize(samplerParams) == SDL_NUM_METAL_SAMPLERS);
-
-        data.mtlsamplers = [[NSMutableArray<id<MTLSamplerState>> alloc] init];
-        samplerdesc = [[MTLSamplerDescriptor alloc] init];
-        for (int i = 0; i < SDL_arraysize(samplerParams); ++i) {
-            samplerdesc.minFilter = samplerParams[i].filter;
-            samplerdesc.magFilter = samplerParams[i].filter;
-            samplerdesc.sAddressMode = samplerParams[i].address;
-            samplerdesc.tAddressMode = samplerParams[i].address;
-            [data.mtlsamplers addObject:[data.mtldevice newSamplerStateWithDescriptor:samplerdesc]];
-        }
+        data.mtlsamplers = [[NSMutableDictionary<NSNumber *, id<MTLSamplerState>> alloc] init];
 
         mtlbufconstantstaging = [data.mtldevice newBufferWithLength:CONSTANTS_LENGTH options:MTLResourceStorageModeShared];