From 636489286af3b1b55cc4b1b02e9b708cc3b807ac Mon Sep 17 00:00:00 2001 From: Michael Sharp Date: Fri, 13 Oct 2023 01:31:22 -0600 Subject: [PATCH 01/19] resolved merge conflicts --- .../System.Numerics.Tensors.sln | 24 ++++++++++++------- .../Numerics/Tensors/TensorPrimitives.cs | 5 ++++ .../Tensors/TensorPrimitives.netcore.cs | 3 ++- .../tests/TensorPrimitivesTests.cs | 17 +++++++++++++ 4 files changed, 40 insertions(+), 9 deletions(-) diff --git a/src/libraries/System.Numerics.Tensors/System.Numerics.Tensors.sln b/src/libraries/System.Numerics.Tensors/System.Numerics.Tensors.sln index 5a1bf792736d94..554f83c7498ac1 100644 --- a/src/libraries/System.Numerics.Tensors/System.Numerics.Tensors.sln +++ b/src/libraries/System.Numerics.Tensors/System.Numerics.Tensors.sln @@ -1,4 +1,8 @@ -Microsoft Visual Studio Solution File, Format Version 12.00 + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio Version 17 +VisualStudioVersion = 17.8.34112.27 +MinimumVisualStudioVersion = 10.0.40219.1 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "TestUtilities", "..\Common\tests\TestUtilities\TestUtilities.csproj", "{9F20CEA1-2216-4432-BBBD-F01E05D17F23}" EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Bcl.Numerics", "..\Microsoft.Bcl.Numerics\ref\Microsoft.Bcl.Numerics.csproj", "{D311ABE4-10A9-4BB1-89CE-6358C55501A8}" @@ -33,11 +37,11 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{AB415F5A-75E EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "gen", "gen", "{083161E5-6049-4D84-9739-9D7797D7117D}" EndProject -Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "gen", "tools\gen", "{841A2FA4-A95F-4612-A8B9-AD2EF769BC71}" +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "gen", "gen", "{841A2FA4-A95F-4612-A8B9-AD2EF769BC71}" EndProject -Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "tools\src", "{DF0561A1-3AB8-4B51-AFB4-392EE1DD6247}" +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{DF0561A1-3AB8-4B51-AFB4-392EE1DD6247}" EndProject -Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "ref", "tools\ref", "{7AC4B2C7-A55C-4C4F-9B02-77F5CBFFF4AB}" +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "ref", "ref", "{7AC4B2C7-A55C-4C4F-9B02-77F5CBFFF4AB}" EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "tools", "tools", "{F9C2AAB1-C7B0-4E43-BB18-4FB16F6E272B}" EndProject @@ -105,23 +109,27 @@ Global EndGlobalSection GlobalSection(NestedProjects) = preSolution {9F20CEA1-2216-4432-BBBD-F01E05D17F23} = {DE94CA7D-BB10-4865-85A6-6B694631247F} - {4AF6A02D-82C8-4898-9EDF-01F107C25061} = {DE94CA7D-BB10-4865-85A6-6B694631247F} {D311ABE4-10A9-4BB1-89CE-6358C55501A8} = {6BC42E6D-848C-4533-B715-F116E7DB3610} - {21CB448A-3882-4337-B416-D1A3E0BCFFC5} = {6BC42E6D-848C-4533-B715-F116E7DB3610} {1578185F-C4FA-4866-936B-E62AAEDD03B7} = {AB415F5A-75E5-4E03-8A92-15CEDEC4CD3A} + {21CB448A-3882-4337-B416-D1A3E0BCFFC5} = {6BC42E6D-848C-4533-B715-F116E7DB3610} {848DD000-3D22-4A25-A9D9-05AFF857A116} = {AB415F5A-75E5-4E03-8A92-15CEDEC4CD3A} + {4AF6A02D-82C8-4898-9EDF-01F107C25061} = {DE94CA7D-BB10-4865-85A6-6B694631247F} {4588351F-4233-4957-B84C-7F8E22B8888A} = {083161E5-6049-4D84-9739-9D7797D7117D} {DB954E01-898A-4FE2-A3AA-180D041AB08F} = {083161E5-6049-4D84-9739-9D7797D7117D} {04FC0651-B9D0-448A-A28B-11B1D4A897F4} = {841A2FA4-A95F-4612-A8B9-AD2EF769BC71} {683A7D28-CC55-4375-848D-E659075ECEE4} = {841A2FA4-A95F-4612-A8B9-AD2EF769BC71} - {841A2FA4-A95F-4612-A8B9-AD2EF769BC71} = {F9C2AAB1-C7B0-4E43-BB18-4FB16F6E272B} {1CBEAEA8-2CA1-4B07-9930-35A785205852} = {DF0561A1-3AB8-4B51-AFB4-392EE1DD6247} {BA7828B1-7953-47A0-AE5A-E22B501C4BD0} = {DF0561A1-3AB8-4B51-AFB4-392EE1DD6247} - {DF0561A1-3AB8-4B51-AFB4-392EE1DD6247} = {F9C2AAB1-C7B0-4E43-BB18-4FB16F6E272B} {57E57290-3A6A-43F8-8764-D4DC8151F89C} = {7AC4B2C7-A55C-4C4F-9B02-77F5CBFFF4AB} + {841A2FA4-A95F-4612-A8B9-AD2EF769BC71} = {F9C2AAB1-C7B0-4E43-BB18-4FB16F6E272B} + {DF0561A1-3AB8-4B51-AFB4-392EE1DD6247} = {F9C2AAB1-C7B0-4E43-BB18-4FB16F6E272B} {7AC4B2C7-A55C-4C4F-9B02-77F5CBFFF4AB} = {F9C2AAB1-C7B0-4E43-BB18-4FB16F6E272B} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {10A5F2C3-5230-4916-9D4D-BBDB94851037} EndGlobalSection + GlobalSection(SharedMSBuildProjectFiles) = preSolution + ..\..\tools\illink\src\ILLink.Shared\ILLink.Shared.projitems*{683a7d28-cc55-4375-848d-e659075ecee4}*SharedItemsImports = 5 + ..\..\tools\illink\src\ILLink.Shared\ILLink.Shared.projitems*{ba7828b1-7953-47a0-ae5a-e22b501c4bd0}*SharedItemsImports = 5 + EndGlobalSection EndGlobal diff --git a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/TensorPrimitives.cs b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/TensorPrimitives.cs index 6bda8b2c900e16..6889337ac85570 100644 --- a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/TensorPrimitives.cs +++ b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/TensorPrimitives.cs @@ -316,6 +316,10 @@ public static void Exp(ReadOnlySpan x, Span destination) => /// public static int IndexOfMax(ReadOnlySpan x) { +#if NET8_0_OR_GREATER + return IndexOfMinMaxCore(x); +#else + int result = -1; if (!x.IsEmpty) @@ -349,6 +353,7 @@ public static int IndexOfMax(ReadOnlySpan x) } return result; +#endif } /// Searches for the index of the single-precision floating-point number with the largest magnitude in the specified tensor. diff --git a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/TensorPrimitives.netcore.cs b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/TensorPrimitives.netcore.cs index 954f4924d81c63..6a53f8ce9ec0cd 100644 --- a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/TensorPrimitives.netcore.cs +++ b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/TensorPrimitives.netcore.cs @@ -8,6 +8,7 @@ using System.Runtime.Intrinsics.Arm; using System.Runtime.Intrinsics.X86; using System.Security.Cryptography; +using System.Threading.Tasks; namespace System.Numerics.Tensors { @@ -1048,7 +1049,7 @@ private static float MinMaxCore(ReadOnlySpan x) if (Vector512.IsHardwareAccelerated && x.Length >= Vector512.Count) { ref float xRef = ref MemoryMarshal.GetReference(x); - + Vector512 minIndex = Vector512.Create(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); // Load the first vector as the initial set of results, and bail immediately // to scalar handling if it contains any NaNs (which don't compare equally to themselves). Vector512 result = Vector512.LoadUnsafe(ref xRef, 0), current; diff --git a/src/libraries/System.Numerics.Tensors/tests/TensorPrimitivesTests.cs b/src/libraries/System.Numerics.Tensors/tests/TensorPrimitivesTests.cs index ed5317420ae87f..5b8a74917a5013 100644 --- a/src/libraries/System.Numerics.Tensors/tests/TensorPrimitivesTests.cs +++ b/src/libraries/System.Numerics.Tensors/tests/TensorPrimitivesTests.cs @@ -990,6 +990,8 @@ public static void IndexOfMax_FirstNaNReturned(int tensorLength) [Fact] public static void IndexOfMax_Negative0LesserThanPositive0() { + System.Diagnostics.Debugger.Launch(); + Assert.Equal(1, TensorPrimitives.IndexOfMax([-0f, +0f])); Assert.Equal(0, TensorPrimitives.IndexOfMax([-0f, -0f, -0f, -0f])); Assert.Equal(4, TensorPrimitives.IndexOfMax([-0f, -0f, -0f, -0f, +0f, +0f, +0f])); @@ -1314,6 +1316,21 @@ public static void Max_Tensor_SpecialValues(int tensorLength) }, x); } + [Fact] + public static void Max_Tensor_Temp() + { + using BoundedMemory x = CreateAndFillTensor(128); + + Assert.Equal(Enumerable.Max(MemoryMarshal.ToEnumerable(x.Memory)), TensorPrimitives.Max(x)); + + float max = float.NegativeInfinity; + foreach (float f in x.Span) + { + max = Math.Max(max, f); + } + Assert.Equal(max, TensorPrimitives.Max(x)); + } + [Theory] [MemberData(nameof(TensorLengths))] public static void Max_Tensor_NanReturned(int tensorLength) From 5eb6c913cd3db7f3ee22f6d5cd138d47c4d5b5d1 Mon Sep 17 00:00:00 2001 From: Michael Sharp Date: Fri, 13 Oct 2023 06:24:15 -0600 Subject: [PATCH 02/19] net core full done --- .../src/ReferenceAssemblyExclusions.txt | 2 + .../Numerics/Tensors/TensorPrimitives.cs | 14 +- .../Tensors/TensorPrimitives.netcore.cs | 1033 ++++++++++++++++- .../tests/TensorPrimitivesTests.cs | 18 + 4 files changed, 1064 insertions(+), 3 deletions(-) create mode 100644 src/libraries/System.Numerics.Tensors/src/ReferenceAssemblyExclusions.txt diff --git a/src/libraries/System.Numerics.Tensors/src/ReferenceAssemblyExclusions.txt b/src/libraries/System.Numerics.Tensors/src/ReferenceAssemblyExclusions.txt new file mode 100644 index 00000000000000..a8f2d0192cfec9 --- /dev/null +++ b/src/libraries/System.Numerics.Tensors/src/ReferenceAssemblyExclusions.txt @@ -0,0 +1,2 @@ +M:System.Numerics.Tensors.TensorPrimitives.ConvertToHalf(System.ReadOnlySpan{System.Single},System.Span{System.Half}) +M:System.Numerics.Tensors.TensorPrimitives.ConvertToSingle(System.ReadOnlySpan{System.Half},System.Span{System.Single}) \ No newline at end of file diff --git a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/TensorPrimitives.cs b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/TensorPrimitives.cs index 6889337ac85570..3578ac8e44ca83 100644 --- a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/TensorPrimitives.cs +++ b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/TensorPrimitives.cs @@ -317,7 +317,7 @@ public static void Exp(ReadOnlySpan x, Span destination) => public static int IndexOfMax(ReadOnlySpan x) { #if NET8_0_OR_GREATER - return IndexOfMinMaxCore(x); + return IndexOfMinMaxCore(x); #else int result = -1; @@ -372,6 +372,9 @@ public static int IndexOfMax(ReadOnlySpan x) /// public static int IndexOfMaxMagnitude(ReadOnlySpan x) { +#if NET8_0_OR_GREATER + return IndexOfMinMaxCore(x); +#else int result = -1; if (!x.IsEmpty) @@ -409,6 +412,7 @@ public static int IndexOfMaxMagnitude(ReadOnlySpan x) } return result; +#endif } /// Searches for the index of the smallest single-precision floating-point number in the specified tensor. @@ -426,6 +430,9 @@ public static int IndexOfMaxMagnitude(ReadOnlySpan x) /// public static int IndexOfMin(ReadOnlySpan x) { +#if NET8_0_OR_GREATER + return IndexOfMinMaxCore(x); +#else int result = -1; if (!x.IsEmpty) @@ -459,6 +466,7 @@ public static int IndexOfMin(ReadOnlySpan x) } return result; +#endif } /// Searches for the index of the single-precision floating-point number with the smallest magnitude in the specified tensor. @@ -477,6 +485,9 @@ public static int IndexOfMin(ReadOnlySpan x) /// public static int IndexOfMinMagnitude(ReadOnlySpan x) { +#if NET8_0_OR_GREATER + return IndexOfMinMaxCore(x); +#else int result = -1; if (!x.IsEmpty) @@ -514,6 +525,7 @@ public static int IndexOfMinMagnitude(ReadOnlySpan x) } return result; +#endif } /// Computes the element-wise natural (base e) logarithm of single-precision floating-point numbers in the specified tensor. diff --git a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/TensorPrimitives.netcore.cs b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/TensorPrimitives.netcore.cs index 6a53f8ce9ec0cd..477c352e7d5155 100644 --- a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/TensorPrimitives.netcore.cs +++ b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/TensorPrimitives.netcore.cs @@ -7,8 +7,6 @@ using System.Runtime.Intrinsics; using System.Runtime.Intrinsics.Arm; using System.Runtime.Intrinsics.X86; -using System.Security.Cryptography; -using System.Threading.Tasks; namespace System.Numerics.Tensors { @@ -1214,6 +1212,197 @@ private static float MinMaxCore(ReadOnlySpan x) } } + /// + /// This is the same as , + /// except it early exits on NaN. + /// + private static int IndexOfMinMaxCore(ReadOnlySpan x) where TIndexOfMinMax : struct, IIndexOfOperator + { + if (x.IsEmpty) + { + return -1; + } + + // This matches the IEEE 754:2019 `maximum`/`minimum` functions. + // It propagates NaN inputs back to the caller and + // otherwise returns the greater of the inputs. + // It treats +0 as greater than -0 as per the specification. + +#if NET8_0_OR_GREATER + if (Vector512.IsHardwareAccelerated && x.Length >= Vector512.Count) + { + ref float xRef = ref MemoryMarshal.GetReference(x); + Vector512 maxIndex = Vector512.Create(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + Vector512 curIndex = Vector512.Create(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + Vector512 increment = Vector512.Create(16); + + // Load the first vector as the initial set of results, and bail immediately + // to scalar handling if it contains any NaNs (which don't compare equally to themselves). + Vector512 result = Vector512.LoadUnsafe(ref xRef, 0), current; + if (!Vector512.EqualsAll(result, result)) + { + return GetFirstNaNIndex(result, maxIndex); + } + + int oneVectorFromEnd = x.Length - Vector512.Count; + int i = Vector512.Count; + + // Aggregate additional vectors into the result as long as there's at least one full vector left to process. + while (i <= oneVectorFromEnd) + { + // Load the next vector, and early exit on NaN. + current = Vector512.LoadUnsafe(ref xRef, (uint)i); + curIndex = Vector512.Add(curIndex, increment); + + if (!Vector512.EqualsAll(current, current)) + { + return GetFirstNaNIndex(current, curIndex); + } + + TIndexOfMinMax.Invoke(ref result, current, ref maxIndex, curIndex); + i += Vector512.Count; + } + + // If any elements remain, handle them in one final vector. + if (i != x.Length) + { + current = Vector512.LoadUnsafe(ref xRef, (uint)(x.Length - Vector512.Count)); + curIndex = Vector512.Add(curIndex, Vector512.Create(x.Length - i)); + + if (!Vector512.EqualsAll(current, current)) + { + return GetFirstNaNIndex(current, curIndex); + } + + Vector512 remainderMask = CreateRemainderMaskSingleVector512(x.Length - i); + + TIndexOfMinMax.Invoke(ref result, current, ref maxIndex, curIndex, remainderMask); + } + + // Aggregate the lanes in the vector to create the final scalar result. + + return TIndexOfMinMax.Invoke(result, maxIndex); + } +#endif + + if (Vector256.IsHardwareAccelerated && x.Length >= Vector256.Count) + { + ref float xRef = ref MemoryMarshal.GetReference(x); + Vector256 maxIndex = Vector256.Create(0, 1, 2, 3, 4, 5, 6, 7); + Vector256 curIndex = Vector256.Create(0, 1, 2, 3, 4, 5, 6, 7); + Vector256 increment = Vector256.Create(8); + + // Load the first vector as the initial set of results, and bail immediately + // to scalar handling if it contains any NaNs (which don't compare equally to themselves). + Vector256 max = Vector256.LoadUnsafe(ref xRef, 0), current; + if (!Vector256.EqualsAll(max, max)) + { + return GetFirstNaNIndex(max, maxIndex); + } + + int oneVectorFromEnd = x.Length - Vector256.Count; + int i = Vector256.Count; + + // Aggregate additional vectors into the result as long as there's at least one full vector left to process. + while (i <= oneVectorFromEnd) + { + // Load the next vector, and early exit on NaN. + current = Vector256.LoadUnsafe(ref xRef, (uint)i); + curIndex = Vector256.Add(curIndex, increment); + + if (!Vector256.EqualsAll(current, current)) + { + return GetFirstNaNIndex(current, curIndex); + } + + TIndexOfMinMax.Invoke(ref max, current, ref maxIndex, curIndex); + + i += Vector256.Count; + } + + // If any elements remain, handle them in one final vector. + if (i != x.Length) + { + current = Vector256.LoadUnsafe(ref xRef, (uint)(x.Length - Vector256.Count)); + curIndex = Vector256.Add(curIndex, Vector256.Create(x.Length - i)); + + if (!Vector256.EqualsAll(current, current)) + { + return GetFirstNaNIndex(current, curIndex); + } + + Vector256 remainderMask = CreateRemainderMaskSingleVector256(x.Length - i); + + TIndexOfMinMax.Invoke(ref max, current, ref maxIndex, curIndex, remainderMask); + + } + + // Aggregate the lanes in the vector to create the final scalar result. + return TIndexOfMinMax.Invoke(max, maxIndex); + } + + if (Vector128.IsHardwareAccelerated && x.Length >= Vector128.Count) + { + ref float xRef = ref MemoryMarshal.GetReference(x); + Vector128 maxIndex = Vector128.Create(0, 1, 2, 3); + Vector128 curIndex = Vector128.Create(0, 1, 2, 3); + Vector128 increment = Vector128.Create(4); + + // Load the first vector as the initial set of results, and bail immediately + // to scalar handling if it contains any NaNs (which don't compare equally to themselves). + Vector128 result = Vector128.LoadUnsafe(ref xRef, 0), current; + if (!Vector128.EqualsAll(result, result)) + { + return GetFirstNaNIndex(result, maxIndex); + } + + int oneVectorFromEnd = x.Length - Vector128.Count; + int i = Vector128.Count; + + // Aggregate additional vectors into the result as long as there's at least one full vector left to process. + while (i <= oneVectorFromEnd) + { + // Load the next vector, and early exit on NaN. + current = Vector128.LoadUnsafe(ref xRef, (uint)i); + curIndex = Vector128.Add(curIndex, increment); + + if (!Vector128.EqualsAll(current, current)) + { + return GetFirstNaNIndex(current, curIndex); + } + + TIndexOfMinMax.Invoke(ref result, current, ref maxIndex, curIndex); + + i += Vector128.Count; + } + + // If any elements remain, handle them in one final vector. + if (i != x.Length) + { + curIndex = Vector128.Add(curIndex, Vector128.Create(x.Length - i)); + + current = Vector128.LoadUnsafe(ref xRef, (uint)(x.Length - Vector128.Count)); + if (!Vector128.EqualsAll(current, current)) + { + return GetFirstNaNIndex(current, curIndex); + } + + Vector128 remainderMask = CreateRemainderMaskSingleVector128(x.Length - i); + + TIndexOfMinMax.Invoke(ref result, current, ref maxIndex, curIndex, remainderMask); + + } + + // Aggregate the lanes in the vector to create the final scalar result. + return TIndexOfMinMax.Invoke(result, maxIndex); + } + + // Scalar path used when either vectorization is not supported or the input is too small to vectorize. + { + return TIndexOfMinMax.Invoke(x); + } + } + /// Performs an element-wise operation on and writes the results to . /// Specifies the operation to perform on each element loaded from . private static void InvokeSpanIntoSpan( @@ -2904,6 +3093,26 @@ private static Vector512 IsNegative(Vector512 vector) => Vector512.LessThan(vector.AsInt32(), Vector512.Zero).AsSingle(); #endif + /// Gets whether the specified is positive. + private static bool IsPositive(float f) => float.IsPositive(f); + + /// Gets whether each specified is positive. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Vector128 IsPositive(Vector128 vector) => + Vector128.GreaterThan(vector.AsInt32(), Vector128.AllBitsSet).AsSingle(); + + /// Gets whether each specified is positive. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Vector256 IsPositive(Vector256 vector) => + Vector256.GreaterThan(vector.AsInt32(), Vector256.AllBitsSet).AsSingle(); + +#if NET8_0_OR_GREATER + /// Gets whether each specified is positive. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Vector512 IsPositive(Vector512 vector) => + Vector512.GreaterThan(vector.AsInt32(), Vector512.AllBitsSet).AsSingle(); +#endif + /// Finds and returns the first NaN value in . /// The vector must have already been validated to contain a NaN. private static float GetFirstNaN(Vector128 vector) @@ -2912,6 +3121,14 @@ private static float GetFirstNaN(Vector128 vector) return vector.GetElement(BitOperations.TrailingZeroCount((~Vector128.Equals(vector, vector)).ExtractMostSignificantBits())); } + /// Finds and returns the first NaN index value in . + /// The vector must have already been validated to contain a NaN. + private static int GetFirstNaNIndex(Vector128 vector, Vector128 index) + { + Debug.Assert(!Vector128.EqualsAll(vector, vector), "Expected vector to contain a NaN"); + return index.GetElement(BitOperations.TrailingZeroCount((~Vector128.Equals(vector, vector)).ExtractMostSignificantBits())); + } + /// Finds and returns the first NaN value in . /// The vector must have already been validated to contain a NaN. private static float GetFirstNaN(Vector256 vector) @@ -2920,6 +3137,14 @@ private static float GetFirstNaN(Vector256 vector) return vector.GetElement(BitOperations.TrailingZeroCount((~Vector256.Equals(vector, vector)).ExtractMostSignificantBits())); } + /// Finds and returns the first NaN index value in . + /// The vector must have already been validated to contain a NaN. + private static int GetFirstNaNIndex(Vector256 vector, Vector256 index) + { + Debug.Assert(!Vector256.EqualsAll(vector, vector), "Expected vector to contain a NaN"); + return index.GetElement(BitOperations.TrailingZeroCount((~Vector256.Equals(vector, vector)).ExtractMostSignificantBits())); + } + #if NET8_0_OR_GREATER /// Finds and returns the first NaN value in . /// The vector must have already been validated to contain a NaN. @@ -2928,6 +3153,14 @@ private static float GetFirstNaN(Vector512 vector) Debug.Assert(!Vector512.EqualsAll(vector, vector), "Expected vector to contain a NaN"); return vector.GetElement(BitOperations.TrailingZeroCount((~Vector512.Equals(vector, vector)).ExtractMostSignificantBits())); } + + /// Finds and returns the first NaN value in . + /// The vector must have already been validated to contain a NaN. + private static int GetFirstNaNIndex(Vector512 vector, Vector512 index) + { + Debug.Assert(!Vector512.EqualsAll(vector, vector), "Expected vector to contain a NaN"); + return index.GetElement(BitOperations.TrailingZeroCount((~Vector512.Equals(vector, vector)).ExtractMostSignificantBits())); + } #endif /// Gets the base 2 logarithm of . @@ -3099,6 +3332,802 @@ public static Vector512 Invoke(Vector512 x, Vector512 y) => #endif } + private interface IIndexOfOperator + { + static abstract int Invoke(ReadOnlySpan result); + static abstract int Invoke(Vector128 result, Vector128 resultIndex); + static abstract void Invoke(ref Vector128 result, Vector128 current, ref Vector128 resultIndex, Vector128 curIndex); + static abstract void Invoke(ref Vector128 result, Vector128 current, ref Vector128 resultIndex, Vector128 curIndex, Vector128 remainderMask); + static abstract int Invoke(Vector256 result, Vector256 resultIndex); + static abstract void Invoke(ref Vector256 result, Vector256 current, ref Vector256 resultIndex, Vector256 curIndex); + static abstract void Invoke(ref Vector256 result, Vector256 current, ref Vector256 resultIndex, Vector256 curIndex, Vector256 remainderMask); +#if NET8_0_OR_GREATER + static abstract int Invoke(Vector512 result, Vector512 resultIndex); + static abstract void Invoke(ref Vector512 result, Vector512 current, ref Vector512 resultIndex, Vector512 curIndex); + static abstract void Invoke(ref Vector512 result, Vector512 current, ref Vector512 resultIndex, Vector512 curIndex, Vector512 remainderMask); +#endif + + static virtual float IdentityValue => throw new NotSupportedException(); + } + + /// MathF.Max(x, y) (but NaNs may not be propagated) + private readonly struct IndexOfMaxOperator : IIndexOfOperator + { + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static int Invoke(ReadOnlySpan result) + { + float curMax = result[0]; + int curIn = 0; + if (float.IsNaN(curMax)) + { + return curIn; + } + + for (int i = 1; i < result.Length; i++) + { + float current = result[i]; + if (float.IsNaN(current)) + { + return i; + } + + if (curMax == current) + { + if (IsNegative(curMax) && !IsNegative(current)) + { + curMax = current; + curIn = i; + } + } + else if (current > curMax) + { + curMax = current; + curIn = i; + } + } + + return curIn; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static int Invoke(Vector128 result, Vector128 maxIndex) + { + float curMax = float.MinValue; + int curIn = 0; + for (int i = 0; i < Vector128.Count; i++) + { + if (result[i] == curMax && IsNegative(curMax) && !IsNegative(result[i])) + { + curMax = result[i]; + curIn = maxIndex[i]; + } + else if (result[i] > curMax) + { + curMax = result[i]; + curIn = maxIndex[i]; + } + } + return curIn; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static void Invoke(ref Vector128 max, Vector128 current, ref Vector128 maxIndex, Vector128 curIndex) + { + maxIndex = Vector128.ConditionalSelect(Vector128.Equals(max, current).AsInt32(), + Vector128.ConditionalSelect(IsNegative(max).AsInt32(), curIndex, maxIndex), + Vector128.ConditionalSelect(Vector128.GreaterThan(max, current).AsInt32(), maxIndex, curIndex)); + + max = Vector128.ConditionalSelect(Vector128.Equals(max, current), + Vector128.ConditionalSelect(IsNegative(max), current, max), + Vector128.Max(max, current)); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static void Invoke(ref Vector128 max, Vector128 current, ref Vector128 maxIndex, Vector128 curIndex, Vector128 remainderMask) + { + maxIndex = Vector128.ConditionalSelect( + Vector128.Equals(remainderMask.AsInt32(), Vector128.Zero), + maxIndex, + Vector128.ConditionalSelect(Vector128.Equals(max, current).AsInt32(), + Vector128.ConditionalSelect(IsNegative(max).AsInt32(), curIndex, maxIndex), + Vector128.ConditionalSelect(Vector128.GreaterThan(max, current).AsInt32(), maxIndex, curIndex))); + + max = Vector128.ConditionalSelect( + Vector128.Equals(remainderMask, Vector128.Zero), + max, + MaxOperator.Invoke(max, current)); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static int Invoke(Vector256 result, Vector256 maxIndex) + { + float curMax = float.MinValue; + int curIn = 0; + for (int i = 0; i < Vector256.Count; i++) + { + if (result[i] == curMax && IsNegative(curMax) && !IsNegative(result[i])) + { + curMax = result[i]; + curIn = maxIndex[i]; + } + else if (result[i] > curMax) + { + curMax = result[i]; + curIn = maxIndex[i]; + } + } + return curIn; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static void Invoke(ref Vector256 max, Vector256 current, ref Vector256 maxIndex, Vector256 curIndex) + { + maxIndex = Vector256.ConditionalSelect(Vector256.Equals(max, current).AsInt32(), + Vector256.ConditionalSelect(IsNegative(max).AsInt32(), curIndex, maxIndex), + Vector256.ConditionalSelect(Vector256.GreaterThan(max, current).AsInt32(), maxIndex, curIndex)); + + max = Vector256.ConditionalSelect(Vector256.Equals(max, current), + Vector256.ConditionalSelect(IsNegative(max), current, max), + Vector256.Max(max, current)); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static void Invoke(ref Vector256 max, Vector256 current, ref Vector256 maxIndex, Vector256 curIndex, Vector256 remainderMask) + { + maxIndex = Vector256.ConditionalSelect( + Vector256.Equals(remainderMask.AsInt32(), Vector256.Zero), + maxIndex, + Vector256.ConditionalSelect(Vector256.Equals(max, current).AsInt32(), + Vector256.ConditionalSelect(IsNegative(max).AsInt32(), curIndex, maxIndex), + Vector256.ConditionalSelect(Vector256.GreaterThan(max, current).AsInt32(), maxIndex, curIndex))); + + max = Vector256.ConditionalSelect( + Vector256.Equals(remainderMask, Vector256.Zero), + max, + MaxOperator.Invoke(max, current)); + } + +#if NET8_0_OR_GREATER + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static int Invoke(Vector512 result, Vector512 maxIndex) + { + float curMax = float.MinValue; + int curIn = 0; + for (int i = 0; i < Vector512.Count; i++) + { + if (result[i] == curMax && IsNegative(curMax) && !IsNegative(result[i])) + { + curMax = result[i]; + curIn = maxIndex[i]; + } + else if (result[i] > curMax) + { + curMax = result[i]; + curIn = maxIndex[i]; + } + } + return curIn; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static void Invoke(ref Vector512 max, Vector512 current, ref Vector512 maxIndex, Vector512 curIndex) + { + maxIndex = Vector512.ConditionalSelect(Vector512.Equals(max, current).AsInt32(), + Vector512.ConditionalSelect(IsNegative(max).AsInt32(), curIndex, maxIndex), + Vector512.ConditionalSelect(Vector512.GreaterThan(max, current).AsInt32(), maxIndex, curIndex)); + + max = Vector512.ConditionalSelect(Vector512.Equals(max, current), + Vector512.ConditionalSelect(IsNegative(max), current, max), + Vector512.Max(max, current)); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static void Invoke(ref Vector512 max, Vector512 current, ref Vector512 maxIndex, Vector512 curIndex, Vector512 remainderMask) + { + maxIndex = Vector512.ConditionalSelect( + Vector512.Equals(remainderMask.AsInt32(), Vector512.Zero), + maxIndex, + Vector512.ConditionalSelect(Vector512.Equals(max, current).AsInt32(), + Vector512.ConditionalSelect(IsNegative(max).AsInt32(), curIndex, maxIndex), + Vector512.ConditionalSelect(Vector512.GreaterThan(max, current).AsInt32(), maxIndex, curIndex))); + + max = Vector512.ConditionalSelect( + Vector512.Equals(remainderMask, Vector512.Zero), + max, + MaxOperator.Invoke(max, current)); + } +#endif + } + + private readonly struct IndexOfMaxMagnitudeOperator : IIndexOfOperator + { + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static int Invoke(ReadOnlySpan result) + { + float curMax = result[0]; + int curIn = 0; + if (float.IsNaN(curMax)) + { + return curIn; + } + + for (int i = 1; i < result.Length; i++) + { + float current = result[i]; + if (float.IsNaN(current)) + { + return i; + } + + if (MathF.Abs(curMax) == MathF.Abs(current)) + { + if (IsNegative(curMax) && !IsNegative(current)) + { + curMax = current; + curIn = i; + } + } + else if (MathF.Abs(current) > MathF.Abs(curMax)) + { + curMax = current; + curIn = i; + } + } + + return curIn; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static int Invoke(Vector128 result, Vector128 maxIndex) + { + float curMax = result[0]; + int curIn = maxIndex[0]; + for (int i = 1; i < Vector128.Count; i++) + { + if (MathF.Abs(result[i]) == MathF.Abs(curMax) && IsNegative(curMax) && !IsNegative(result[i])) + { + curMax = result[i]; + curIn = maxIndex[i]; + } + else if (MathF.Abs(result[i]) > MathF.Abs(curMax)) + { + curMax = result[i]; + curIn = maxIndex[i]; + } + } + return curIn; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static void Invoke(ref Vector128 max, Vector128 current, ref Vector128 maxIndex, Vector128 curIndex) + { + Vector128 maxMag = Vector128.Abs(max), currentMag = Vector128.Abs(current); + + maxIndex = Vector128.ConditionalSelect(Vector128.Equals(max, current).AsInt32(), + Vector128.ConditionalSelect(IsNegative(max).AsInt32(), curIndex, maxIndex), + Vector128.ConditionalSelect(Vector128.GreaterThan(maxMag, currentMag).AsInt32(), maxIndex, curIndex)); + + max = Vector128.ConditionalSelect(Vector128.Equals(max, current), + Vector128.ConditionalSelect(IsNegative(max), current, max), + Vector128.Max(max, current)); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static void Invoke(ref Vector128 max, Vector128 current, ref Vector128 maxIndex, Vector128 curIndex, Vector128 remainderMask) + { + Vector128 maxMag = Vector128.Abs(max), currentMag = Vector128.Abs(current); + + maxIndex = Vector128.ConditionalSelect( + Vector128.Equals(remainderMask.AsInt32(), Vector128.Zero), + maxIndex, + Vector128.ConditionalSelect(Vector128.Equals(max, current).AsInt32(), + Vector128.ConditionalSelect(IsNegative(max).AsInt32(), curIndex, maxIndex), + Vector128.ConditionalSelect(Vector128.GreaterThan(maxMag, currentMag).AsInt32(), maxIndex, curIndex))); + + max = Vector128.ConditionalSelect( + Vector128.Equals(remainderMask, Vector128.Zero), + max, + MaxOperator.Invoke(max, current)); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static int Invoke(Vector256 result, Vector256 maxIndex) + { + float curMax = result[0]; + int curIn = maxIndex[0]; + for (int i = 1; i < Vector256.Count; i++) + { + if (MathF.Abs(result[i]) == MathF.Abs(curMax) && IsNegative(curMax) && !IsNegative(result[i])) + { + curMax = result[i]; + curIn = maxIndex[i]; + } + else if (MathF.Abs(result[i]) > MathF.Abs(curMax)) + { + curMax = result[i]; + curIn = maxIndex[i]; + } + } + return curIn; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static void Invoke(ref Vector256 max, Vector256 current, ref Vector256 maxIndex, Vector256 curIndex) + { + Vector256 maxMag = Vector256.Abs(max), currentMag = Vector256.Abs(current); + + maxIndex = Vector256.ConditionalSelect(Vector256.Equals(max, current).AsInt32(), + Vector256.ConditionalSelect(IsNegative(max).AsInt32(), curIndex, maxIndex), + Vector256.ConditionalSelect(Vector256.GreaterThan(maxMag, currentMag).AsInt32(), maxIndex, curIndex)); + + max = Vector256.ConditionalSelect(Vector256.Equals(max, current), + Vector256.ConditionalSelect(IsNegative(max), current, max), + Vector256.Max(max, current)); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static void Invoke(ref Vector256 max, Vector256 current, ref Vector256 maxIndex, Vector256 curIndex, Vector256 remainderMask) + { + Vector256 maxMag = Vector256.Abs(max), currentMag = Vector256.Abs(current); + + maxIndex = Vector256.ConditionalSelect( + Vector256.Equals(remainderMask.AsInt32(), Vector256.Zero), + maxIndex, + Vector256.ConditionalSelect(Vector256.Equals(max, current).AsInt32(), + Vector256.ConditionalSelect(IsNegative(max).AsInt32(), curIndex, maxIndex), + Vector256.ConditionalSelect(Vector256.GreaterThan(maxMag, currentMag).AsInt32(), maxIndex, curIndex))); + + max = Vector256.ConditionalSelect( + Vector256.Equals(remainderMask, Vector256.Zero), + max, + MaxOperator.Invoke(max, current)); + } + +#if NET8_0_OR_GREATER + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static int Invoke(Vector512 result, Vector512 maxIndex) + { + float curMax = result[0]; + int curIn = maxIndex[0]; + for (int i = 1; i < Vector512.Count; i++) + { + if (MathF.Abs(result[i]) == MathF.Abs(curMax) && IsNegative(curMax) && !IsNegative(result[i])) + { + curMax = result[i]; + curIn = maxIndex[i]; + } + else if (MathF.Abs(result[i]) > MathF.Abs(curMax)) + { + curMax = result[i]; + curIn = maxIndex[i]; + } + } + return curIn; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static void Invoke(ref Vector512 max, Vector512 current, ref Vector512 maxIndex, Vector512 curIndex) + { + Vector512 maxMag = Vector512.Abs(max), currentMag = Vector512.Abs(current); + + maxIndex = Vector512.ConditionalSelect(Vector512.Equals(max, current).AsInt32(), + Vector512.ConditionalSelect(IsNegative(max).AsInt32(), curIndex, maxIndex), + Vector512.ConditionalSelect(Vector512.GreaterThan(maxMag, currentMag).AsInt32(), maxIndex, curIndex)); + + max = Vector512.ConditionalSelect(Vector512.Equals(max, current), + Vector512.ConditionalSelect(IsNegative(max), current, max), + Vector512.Max(maxMag, currentMag)); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static void Invoke(ref Vector512 max, Vector512 current, ref Vector512 maxIndex, Vector512 curIndex, Vector512 remainderMask) + { + Vector512 maxMag = Vector512.Abs(max), currentMag = Vector512.Abs(current); + + maxIndex = Vector512.ConditionalSelect( + Vector512.Equals(remainderMask.AsInt32(), Vector512.Zero), + maxIndex, + Vector512.ConditionalSelect(Vector512.Equals(max, current).AsInt32(), + Vector512.ConditionalSelect(IsNegative(max).AsInt32(), curIndex, maxIndex), + Vector512.ConditionalSelect(Vector512.GreaterThan(maxMag, currentMag).AsInt32(), maxIndex, curIndex))); + + max = Vector512.ConditionalSelect( + Vector512.Equals(remainderMask, Vector512.Zero), + max, + MaxOperator.Invoke(maxMag, currentMag)); + } +#endif + } + + /// MathF.Min(x, y) (but NaNs may not be propagated) + private readonly struct IndexOfMinOperator : IIndexOfOperator + { + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static int Invoke(ReadOnlySpan result) + { + float curMin = result[0]; + int curIn = 0; + if (float.IsNaN(curMin)) + { + return curIn; + } + + for (int i = 1; i < result.Length; i++) + { + float current = result[i]; + if (float.IsNaN(current)) + { + return i; + } + + if (curMin == current) + { + if (IsPositive(curMin) && !IsPositive(current)) + { + curMin = current; + curIn = i; + } + } + else if (current < curMin) + { + curMin = current; + curIn = i; + } + } + + return curIn; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static int Invoke(Vector128 result, Vector128 resultIndex) + { + float curMin = float.MaxValue; + int curIn = 0; + for (int i = 0; i < Vector128.Count; i++) + { + if (result[i] == curMin && IsPositive(curMin) && !IsPositive(result[i])) + { + curMin = result[i]; + curIn = resultIndex[i]; + } + else if (result[i] < curMin) + { + curMin = result[i]; + curIn = resultIndex[i]; + } + } + return curIn; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static void Invoke(ref Vector128 result, Vector128 current, ref Vector128 resultIndex, Vector128 curIndex) + { + resultIndex = Vector128.ConditionalSelect(Vector128.Equals(result, current).AsInt32(), + Vector128.ConditionalSelect(IsPositive(result).AsInt32(), curIndex, resultIndex), + Vector128.ConditionalSelect(Vector128.LessThan(result, current).AsInt32(), resultIndex, curIndex)); + + result = Vector128.ConditionalSelect(Vector128.Equals(result, current), + Vector128.ConditionalSelect(IsPositive(result), current, result), + Vector128.Min(result, current)); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static void Invoke(ref Vector128 result, Vector128 current, ref Vector128 resultIndex, Vector128 curIndex, Vector128 remainderMask) + { + resultIndex = Vector128.ConditionalSelect( + Vector128.Equals(remainderMask.AsInt32(), Vector128.Zero), + resultIndex, + Vector128.ConditionalSelect(Vector128.Equals(result, current).AsInt32(), + Vector128.ConditionalSelect(IsPositive(result).AsInt32(), curIndex, resultIndex), + Vector128.ConditionalSelect(Vector128.LessThan(result, current).AsInt32(), resultIndex, curIndex))); + + result = Vector128.ConditionalSelect( + Vector128.Equals(remainderMask, Vector128.Zero), + result, + MinOperator.Invoke(result, current)); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static int Invoke(Vector256 result, Vector256 resultIndex) + { + float curMin = float.MaxValue; + int curIn = 0; + for (int i = 0; i < Vector256.Count; i++) + { + if (result[i] == curMin && IsPositive(curMin) && !IsPositive(result[i])) + { + curMin = result[i]; + curIn = resultIndex[i]; + } + else if (result[i] < curMin) + { + curMin = result[i]; + curIn = resultIndex[i]; + } + } + return curIn; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static void Invoke(ref Vector256 result, Vector256 current, ref Vector256 resultIndex, Vector256 curIndex) + { + resultIndex = Vector256.ConditionalSelect(Vector256.Equals(result, current).AsInt32(), + Vector256.ConditionalSelect(IsPositive(result).AsInt32(), curIndex, resultIndex), + Vector256.ConditionalSelect(Vector256.LessThan(result, current).AsInt32(), resultIndex, curIndex)); + + result = Vector256.ConditionalSelect(Vector256.Equals(result, current), + Vector256.ConditionalSelect(IsPositive(result), current, result), + Vector256.Min(result, current)); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static void Invoke(ref Vector256 result, Vector256 current, ref Vector256 resultIndex, Vector256 curIndex, Vector256 remainderMask) + { + resultIndex = Vector256.ConditionalSelect( + Vector256.Equals(remainderMask.AsInt32(), Vector256.Zero), + resultIndex, + Vector256.ConditionalSelect(Vector256.Equals(result, current).AsInt32(), + Vector256.ConditionalSelect(IsPositive(result).AsInt32(), curIndex, resultIndex), + Vector256.ConditionalSelect(Vector256.LessThan(result, current).AsInt32(), resultIndex, curIndex))); + + result = Vector256.ConditionalSelect( + Vector256.Equals(remainderMask, Vector256.Zero), + result, + MinOperator.Invoke(result, current)); + } + +#if NET8_0_OR_GREATER + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static int Invoke(Vector512 result, Vector512 resultIndex) + { + float curMin = float.MaxValue; + int curIn = 0; + for (int i = 0; i < Vector512.Count; i++) + { + if (result[i] == curMin && IsPositive(curMin) && !IsPositive(result[i])) + { + curMin = result[i]; + curIn = resultIndex[i]; + } + else if (result[i] < curMin) + { + curMin = result[i]; + curIn = resultIndex[i]; + } + } + return curIn; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static void Invoke(ref Vector512 result, Vector512 current, ref Vector512 resultIndex, Vector512 curIndex) + { + resultIndex = Vector512.ConditionalSelect(Vector512.Equals(result, current).AsInt32(), + Vector512.ConditionalSelect(IsNegative(result).AsInt32(), curIndex, resultIndex), + Vector512.ConditionalSelect(Vector512.LessThan(result, current).AsInt32(), resultIndex, curIndex)); + + result = Vector512.ConditionalSelect(Vector512.Equals(result, current), + Vector512.ConditionalSelect(IsNegative(result), current, result), + Vector512.Min(result, current)); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static void Invoke(ref Vector512 result, Vector512 current, ref Vector512 resultIndex, Vector512 curIndex, Vector512 remainderMask) + { + resultIndex = Vector512.ConditionalSelect( + Vector512.Equals(remainderMask.AsInt32(), Vector512.Zero), + resultIndex, + Vector512.ConditionalSelect(Vector512.Equals(result, current).AsInt32(), + Vector512.ConditionalSelect(IsPositive(result).AsInt32(), curIndex, resultIndex), + Vector512.ConditionalSelect(Vector512.LessThan(result, current).AsInt32(), resultIndex, curIndex))); + + result = Vector512.ConditionalSelect( + Vector512.Equals(remainderMask, Vector512.Zero), + result, + MinOperator.Invoke(result, current)); + } +#endif + } + + private readonly struct IndexOfMinMagnitudeOperator : IIndexOfOperator + { + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static int Invoke(ReadOnlySpan result) + { + float curMin = result[0]; + int curIn = 0; + if (float.IsNaN(curMin)) + { + return curIn; + } + + for (int i = 1; i < result.Length; i++) + { + float current = result[i]; + if (float.IsNaN(current)) + { + return i; + } + + if (MathF.Abs(curMin) == MathF.Abs(current)) + { + if (IsPositive(curMin) && !IsPositive(current)) + { + curMin = current; + curIn = i; + } + } + else if (MathF.Abs(current) < MathF.Abs(curMin)) + { + curMin = current; + curIn = i; + } + } + + return curIn; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static int Invoke(Vector128 result, Vector128 resultIndex) + { + float curMin = result[0]; + int curIn = resultIndex[0]; + for (int i = 1; i < Vector128.Count; i++) + { + if (MathF.Abs(result[i]) == MathF.Abs(curMin) && IsPositive(curMin) && !IsPositive(result[i])) + { + curMin = result[i]; + curIn = resultIndex[i]; + } + else if (MathF.Abs(result[i]) < MathF.Abs(curMin)) + { + curMin = result[i]; + curIn = resultIndex[i]; + } + } + return curIn; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static void Invoke(ref Vector128 result, Vector128 current, ref Vector128 resultIndex, Vector128 curIndex) + { + Vector128 minMag = Vector128.Abs(result), currentMag = Vector128.Abs(current); + + resultIndex = Vector128.ConditionalSelect(Vector128.Equals(result, current).AsInt32(), + Vector128.ConditionalSelect(IsPositive(result).AsInt32(), curIndex, resultIndex), + Vector128.ConditionalSelect(Vector128.LessThan(minMag, currentMag).AsInt32(), resultIndex, curIndex)); + + result = Vector128.ConditionalSelect(Vector128.Equals(result, current), + Vector128.ConditionalSelect(IsPositive(result), current, result), + Vector128.Min(minMag, currentMag)); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static void Invoke(ref Vector128 result, Vector128 current, ref Vector128 resultIndex, Vector128 curIndex, Vector128 remainderMask) + { + Vector128 minMag = Vector128.Abs(result), currentMag = Vector128.Abs(current); + + resultIndex = Vector128.ConditionalSelect( + Vector128.Equals(remainderMask.AsInt32(), Vector128.Zero), + resultIndex, + Vector128.ConditionalSelect(Vector128.Equals(result, current).AsInt32(), + Vector128.ConditionalSelect(IsPositive(result).AsInt32(), curIndex, resultIndex), + Vector128.ConditionalSelect(Vector128.LessThan(minMag, currentMag).AsInt32(), resultIndex, curIndex))); + + result = Vector128.ConditionalSelect( + Vector128.Equals(remainderMask, Vector128.Zero), + result, + MinOperator.Invoke(minMag, currentMag)); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static int Invoke(Vector256 result, Vector256 resultIndex) + { + float curMin = result[0]; + int curIn = resultIndex[0]; + for (int i = 1; i < Vector256.Count; i++) + { + if (MathF.Abs(result[i]) == MathF.Abs(curMin) && IsPositive(curMin) && !IsPositive(result[i])) + { + curMin = result[i]; + curIn = resultIndex[i]; + } + else if (MathF.Abs(result[i]) < MathF.Abs(curMin)) + { + curMin = result[i]; + curIn = resultIndex[i]; + } + } + return curIn; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static void Invoke(ref Vector256 result, Vector256 current, ref Vector256 resultIndex, Vector256 curIndex) + { + Vector256 minMag = Vector256.Abs(result), currentMag = Vector256.Abs(current); + + resultIndex = Vector256.ConditionalSelect(Vector256.Equals(result, current).AsInt32(), + Vector256.ConditionalSelect(IsPositive(result).AsInt32(), curIndex, resultIndex), + Vector256.ConditionalSelect(Vector256.LessThan(minMag, currentMag).AsInt32(), resultIndex, curIndex)); + + result = Vector256.ConditionalSelect(Vector256.Equals(result, current), + Vector256.ConditionalSelect(IsPositive(result), current, result), + Vector256.Min(minMag, currentMag)); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static void Invoke(ref Vector256 result, Vector256 current, ref Vector256 resultIndex, Vector256 curIndex, Vector256 remainderMask) + { + Vector256 minMag = Vector256.Abs(result), currentMag = Vector256.Abs(current); + + resultIndex = Vector256.ConditionalSelect( + Vector256.Equals(remainderMask.AsInt32(), Vector256.Zero), + resultIndex, + Vector256.ConditionalSelect(Vector256.Equals(result, current).AsInt32(), + Vector256.ConditionalSelect(IsPositive(result).AsInt32(), curIndex, resultIndex), + Vector256.ConditionalSelect(Vector256.LessThan(minMag, currentMag).AsInt32(), resultIndex, curIndex))); + + result = Vector256.ConditionalSelect( + Vector256.Equals(remainderMask, Vector256.Zero), + result, + MinOperator.Invoke(minMag, currentMag)); + } + +#if NET8_0_OR_GREATER + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static int Invoke(Vector512 result, Vector512 resultIndex) + { + float curMin = result[0]; + int curIn = resultIndex[0]; + for (int i = 1; i < Vector512.Count; i++) + { + if (MathF.Abs(result[i]) == MathF.Abs(curMin) && IsPositive(curMin) && !IsPositive(result[i])) + { + curMin = result[i]; + curIn = resultIndex[i]; + } + else if (MathF.Abs(result[i]) < MathF.Abs(curMin)) + { + curMin = result[i]; + curIn = resultIndex[i]; + } + } + return curIn; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static void Invoke(ref Vector512 result, Vector512 current, ref Vector512 resultIndex, Vector512 curIndex) + { + Vector512 minMag = Vector512.Abs(result), currentMag = Vector512.Abs(current); + + resultIndex = Vector512.ConditionalSelect(Vector512.Equals(result, current).AsInt32(), + Vector512.ConditionalSelect(IsNegative(result).AsInt32(), curIndex, resultIndex), + Vector512.ConditionalSelect(Vector512.LessThan(minMag, currentMag).AsInt32(), resultIndex, curIndex)); + + result = Vector512.ConditionalSelect(Vector512.Equals(result, current), + Vector512.ConditionalSelect(IsNegative(result), current, result), + Vector512.Min(minMag, currentMag)); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static void Invoke(ref Vector512 result, Vector512 current, ref Vector512 resultIndex, Vector512 curIndex, Vector512 remainderMask) + { + Vector512 minMag = Vector512.Abs(result), currentMag = Vector512.Abs(current); + + resultIndex = Vector512.ConditionalSelect( + Vector512.Equals(remainderMask.AsInt32(), Vector512.Zero), + resultIndex, + Vector512.ConditionalSelect(Vector512.Equals(result, current).AsInt32(), + Vector512.ConditionalSelect(IsPositive(result).AsInt32(), curIndex, resultIndex), + Vector512.ConditionalSelect(Vector512.LessThan(minMag, currentMag).AsInt32(), resultIndex, curIndex))); + + result = Vector512.ConditionalSelect( + Vector512.Equals(remainderMask, Vector512.Zero), + result, + MinOperator.Invoke(minMag, currentMag)); + } +#endif + } + /// MathF.Max(x, y) private readonly struct MaxPropagateNaNOperator : IBinaryOperator { diff --git a/src/libraries/System.Numerics.Tensors/tests/TensorPrimitivesTests.cs b/src/libraries/System.Numerics.Tensors/tests/TensorPrimitivesTests.cs index 5b8a74917a5013..0f0363043b18f5 100644 --- a/src/libraries/System.Numerics.Tensors/tests/TensorPrimitivesTests.cs +++ b/src/libraries/System.Numerics.Tensors/tests/TensorPrimitivesTests.cs @@ -1113,6 +1113,24 @@ public static void IndexOfMinMagnitude(int tensorLength) } } + [Fact] + public static void IndexOfMinMagnitude_Temp() + { + int tensorLength = 5; + foreach (int expected in new[] { 0, tensorLength / 2, tensorLength - 1 }) + { + using BoundedMemory x = CreateTensor(tensorLength); + for (int i = 0; i < x.Length; i++) + { + x[i] = i % 2 == 0 ? 42 : -42; + } + + x[expected] = -41; + + Assert.Equal(expected, TensorPrimitives.IndexOfMinMagnitude(x)); + } + } + [Theory] [MemberData(nameof(TensorLengths))] public static void IndexOfMinMagnitude_FirstNaNReturned(int tensorLength) From 1886a46b2f41d5e1beb8c35767915c8aaa7f67eb Mon Sep 17 00:00:00 2001 From: Michael Sharp Date: Fri, 13 Oct 2023 06:34:28 -0600 Subject: [PATCH 03/19] minor code cleanup --- .../tests/TensorPrimitivesTests.cs | 35 ------------------- 1 file changed, 35 deletions(-) diff --git a/src/libraries/System.Numerics.Tensors/tests/TensorPrimitivesTests.cs b/src/libraries/System.Numerics.Tensors/tests/TensorPrimitivesTests.cs index 0f0363043b18f5..ed5317420ae87f 100644 --- a/src/libraries/System.Numerics.Tensors/tests/TensorPrimitivesTests.cs +++ b/src/libraries/System.Numerics.Tensors/tests/TensorPrimitivesTests.cs @@ -990,8 +990,6 @@ public static void IndexOfMax_FirstNaNReturned(int tensorLength) [Fact] public static void IndexOfMax_Negative0LesserThanPositive0() { - System.Diagnostics.Debugger.Launch(); - Assert.Equal(1, TensorPrimitives.IndexOfMax([-0f, +0f])); Assert.Equal(0, TensorPrimitives.IndexOfMax([-0f, -0f, -0f, -0f])); Assert.Equal(4, TensorPrimitives.IndexOfMax([-0f, -0f, -0f, -0f, +0f, +0f, +0f])); @@ -1113,24 +1111,6 @@ public static void IndexOfMinMagnitude(int tensorLength) } } - [Fact] - public static void IndexOfMinMagnitude_Temp() - { - int tensorLength = 5; - foreach (int expected in new[] { 0, tensorLength / 2, tensorLength - 1 }) - { - using BoundedMemory x = CreateTensor(tensorLength); - for (int i = 0; i < x.Length; i++) - { - x[i] = i % 2 == 0 ? 42 : -42; - } - - x[expected] = -41; - - Assert.Equal(expected, TensorPrimitives.IndexOfMinMagnitude(x)); - } - } - [Theory] [MemberData(nameof(TensorLengths))] public static void IndexOfMinMagnitude_FirstNaNReturned(int tensorLength) @@ -1334,21 +1314,6 @@ public static void Max_Tensor_SpecialValues(int tensorLength) }, x); } - [Fact] - public static void Max_Tensor_Temp() - { - using BoundedMemory x = CreateAndFillTensor(128); - - Assert.Equal(Enumerable.Max(MemoryMarshal.ToEnumerable(x.Memory)), TensorPrimitives.Max(x)); - - float max = float.NegativeInfinity; - foreach (float f in x.Span) - { - max = Math.Max(max, f); - } - Assert.Equal(max, TensorPrimitives.Max(x)); - } - [Theory] [MemberData(nameof(TensorLengths))] public static void Max_Tensor_NanReturned(int tensorLength) From cc68a7528246a6fea48e5aa3a447bec98fa7e289 Mon Sep 17 00:00:00 2001 From: Michael Sharp Date: Fri, 13 Oct 2023 12:35:40 -0600 Subject: [PATCH 04/19] NetStandard and PR fixes. --- .../Numerics/Tensors/TensorPrimitives.cs | 161 +----- .../Tensors/TensorPrimitives.netcore.cs | 36 +- .../Tensors/TensorPrimitives.netstandard.cs | 485 ++++++++++++++++++ 3 files changed, 512 insertions(+), 170 deletions(-) diff --git a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/TensorPrimitives.cs b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/TensorPrimitives.cs index 3578ac8e44ca83..766c242f1c246a 100644 --- a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/TensorPrimitives.cs +++ b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/TensorPrimitives.cs @@ -316,44 +316,11 @@ public static void Exp(ReadOnlySpan x, Span destination) => /// public static int IndexOfMax(ReadOnlySpan x) { -#if NET8_0_OR_GREATER - return IndexOfMinMaxCore(x); -#else - - int result = -1; - - if (!x.IsEmpty) + if (x.IsEmpty) { - result = 0; - float max = float.NegativeInfinity; - - for (int i = 0; i < x.Length; i++) - { - float current = x[i]; - - if (current != max) - { - if (float.IsNaN(current)) - { - return i; - } - - if (max < current) - { - result = i; - max = current; - } - } - else if (IsNegative(max) && !IsNegative(current)) - { - result = i; - max = current; - } - } + return -1; } - - return result; -#endif + return IndexOfMinMaxCore(x); } /// Searches for the index of the single-precision floating-point number with the largest magnitude in the specified tensor. @@ -372,47 +339,11 @@ public static int IndexOfMax(ReadOnlySpan x) /// public static int IndexOfMaxMagnitude(ReadOnlySpan x) { -#if NET8_0_OR_GREATER - return IndexOfMinMaxCore(x); -#else - int result = -1; - - if (!x.IsEmpty) + if (x.IsEmpty) { - result = 0; - float max = float.NegativeInfinity; - float maxMag = float.NegativeInfinity; - - for (int i = 0; i < x.Length; i++) - { - float current = x[i]; - float currentMag = Math.Abs(current); - - if (currentMag != maxMag) - { - if (float.IsNaN(currentMag)) - { - return i; - } - - if (maxMag < currentMag) - { - result = i; - max = current; - maxMag = currentMag; - } - } - else if (IsNegative(max) && !IsNegative(current)) - { - result = i; - max = current; - maxMag = currentMag; - } - } + return -1; } - - return result; -#endif + return IndexOfMinMaxCore(x); } /// Searches for the index of the smallest single-precision floating-point number in the specified tensor. @@ -430,43 +361,11 @@ public static int IndexOfMaxMagnitude(ReadOnlySpan x) /// public static int IndexOfMin(ReadOnlySpan x) { -#if NET8_0_OR_GREATER - return IndexOfMinMaxCore(x); -#else - int result = -1; - - if (!x.IsEmpty) + if (x.IsEmpty) { - result = 0; - float min = float.PositiveInfinity; - - for (int i = 0; i < x.Length; i++) - { - float current = x[i]; - - if (current != min) - { - if (float.IsNaN(current)) - { - return i; - } - - if (current < min) - { - result = i; - min = current; - } - } - else if (IsNegative(current) && !IsNegative(min)) - { - result = i; - min = current; - } - } + return -1; } - - return result; -#endif + return IndexOfMinMaxCore(x); } /// Searches for the index of the single-precision floating-point number with the smallest magnitude in the specified tensor. @@ -485,47 +384,11 @@ public static int IndexOfMin(ReadOnlySpan x) /// public static int IndexOfMinMagnitude(ReadOnlySpan x) { -#if NET8_0_OR_GREATER - return IndexOfMinMaxCore(x); -#else - int result = -1; - - if (!x.IsEmpty) + if (x.IsEmpty) { - result = 0; - float min = float.PositiveInfinity; - float minMag = float.PositiveInfinity; - - for (int i = 0; i < x.Length; i++) - { - float current = x[i]; - float currentMag = Math.Abs(current); - - if (currentMag != minMag) - { - if (float.IsNaN(currentMag)) - { - return i; - } - - if (currentMag < minMag) - { - result = i; - min = current; - minMag = currentMag; - } - } - else if (IsNegative(current) && !IsNegative(min)) - { - result = i; - min = current; - minMag = currentMag; - } - } + return -1; } - - return result; -#endif + return IndexOfMinMaxCore(x); } /// Computes the element-wise natural (base e) logarithm of single-precision floating-point numbers in the specified tensor. diff --git a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/TensorPrimitives.netcore.cs b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/TensorPrimitives.netcore.cs index 477c352e7d5155..c4a1d81d8cef3d 100644 --- a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/TensorPrimitives.netcore.cs +++ b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/TensorPrimitives.netcore.cs @@ -1047,7 +1047,6 @@ private static float MinMaxCore(ReadOnlySpan x) if (Vector512.IsHardwareAccelerated && x.Length >= Vector512.Count) { ref float xRef = ref MemoryMarshal.GetReference(x); - Vector512 minIndex = Vector512.Create(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16); // Load the first vector as the initial set of results, and bail immediately // to scalar handling if it contains any NaNs (which don't compare equally to themselves). Vector512 result = Vector512.LoadUnsafe(ref xRef, 0), current; @@ -1218,11 +1217,6 @@ private static float MinMaxCore(ReadOnlySpan x) /// private static int IndexOfMinMaxCore(ReadOnlySpan x) where TIndexOfMinMax : struct, IIndexOfOperator { - if (x.IsEmpty) - { - return -1; - } - // This matches the IEEE 754:2019 `maximum`/`minimum` functions. // It propagates NaN inputs back to the caller and // otherwise returns the greater of the inputs. @@ -1232,7 +1226,7 @@ private static int IndexOfMinMaxCore(ReadOnlySpan x) wher if (Vector512.IsHardwareAccelerated && x.Length >= Vector512.Count) { ref float xRef = ref MemoryMarshal.GetReference(x); - Vector512 maxIndex = Vector512.Create(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + Vector512 resultIndex = Vector512.Create(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); Vector512 curIndex = Vector512.Create(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); Vector512 increment = Vector512.Create(16); @@ -1241,7 +1235,7 @@ private static int IndexOfMinMaxCore(ReadOnlySpan x) wher Vector512 result = Vector512.LoadUnsafe(ref xRef, 0), current; if (!Vector512.EqualsAll(result, result)) { - return GetFirstNaNIndex(result, maxIndex); + return GetFirstNaNIndex(result, resultIndex); } int oneVectorFromEnd = x.Length - Vector512.Count; @@ -1259,7 +1253,7 @@ private static int IndexOfMinMaxCore(ReadOnlySpan x) wher return GetFirstNaNIndex(current, curIndex); } - TIndexOfMinMax.Invoke(ref result, current, ref maxIndex, curIndex); + TIndexOfMinMax.Invoke(ref result, current, ref resultIndex, curIndex); i += Vector512.Count; } @@ -1276,19 +1270,19 @@ private static int IndexOfMinMaxCore(ReadOnlySpan x) wher Vector512 remainderMask = CreateRemainderMaskSingleVector512(x.Length - i); - TIndexOfMinMax.Invoke(ref result, current, ref maxIndex, curIndex, remainderMask); + TIndexOfMinMax.Invoke(ref result, current, ref resultIndex, curIndex, remainderMask); } // Aggregate the lanes in the vector to create the final scalar result. - return TIndexOfMinMax.Invoke(result, maxIndex); + return TIndexOfMinMax.Invoke(result, resultIndex); } #endif if (Vector256.IsHardwareAccelerated && x.Length >= Vector256.Count) { ref float xRef = ref MemoryMarshal.GetReference(x); - Vector256 maxIndex = Vector256.Create(0, 1, 2, 3, 4, 5, 6, 7); + Vector256 resultIndex = Vector256.Create(0, 1, 2, 3, 4, 5, 6, 7); Vector256 curIndex = Vector256.Create(0, 1, 2, 3, 4, 5, 6, 7); Vector256 increment = Vector256.Create(8); @@ -1297,7 +1291,7 @@ private static int IndexOfMinMaxCore(ReadOnlySpan x) wher Vector256 max = Vector256.LoadUnsafe(ref xRef, 0), current; if (!Vector256.EqualsAll(max, max)) { - return GetFirstNaNIndex(max, maxIndex); + return GetFirstNaNIndex(max, resultIndex); } int oneVectorFromEnd = x.Length - Vector256.Count; @@ -1315,7 +1309,7 @@ private static int IndexOfMinMaxCore(ReadOnlySpan x) wher return GetFirstNaNIndex(current, curIndex); } - TIndexOfMinMax.Invoke(ref max, current, ref maxIndex, curIndex); + TIndexOfMinMax.Invoke(ref max, current, ref resultIndex, curIndex); i += Vector256.Count; } @@ -1333,18 +1327,18 @@ private static int IndexOfMinMaxCore(ReadOnlySpan x) wher Vector256 remainderMask = CreateRemainderMaskSingleVector256(x.Length - i); - TIndexOfMinMax.Invoke(ref max, current, ref maxIndex, curIndex, remainderMask); + TIndexOfMinMax.Invoke(ref max, current, ref resultIndex, curIndex, remainderMask); } // Aggregate the lanes in the vector to create the final scalar result. - return TIndexOfMinMax.Invoke(max, maxIndex); + return TIndexOfMinMax.Invoke(max, resultIndex); } if (Vector128.IsHardwareAccelerated && x.Length >= Vector128.Count) { ref float xRef = ref MemoryMarshal.GetReference(x); - Vector128 maxIndex = Vector128.Create(0, 1, 2, 3); + Vector128 resultIndex = Vector128.Create(0, 1, 2, 3); Vector128 curIndex = Vector128.Create(0, 1, 2, 3); Vector128 increment = Vector128.Create(4); @@ -1353,7 +1347,7 @@ private static int IndexOfMinMaxCore(ReadOnlySpan x) wher Vector128 result = Vector128.LoadUnsafe(ref xRef, 0), current; if (!Vector128.EqualsAll(result, result)) { - return GetFirstNaNIndex(result, maxIndex); + return GetFirstNaNIndex(result, resultIndex); } int oneVectorFromEnd = x.Length - Vector128.Count; @@ -1371,7 +1365,7 @@ private static int IndexOfMinMaxCore(ReadOnlySpan x) wher return GetFirstNaNIndex(current, curIndex); } - TIndexOfMinMax.Invoke(ref result, current, ref maxIndex, curIndex); + TIndexOfMinMax.Invoke(ref result, current, ref resultIndex, curIndex); i += Vector128.Count; } @@ -1389,12 +1383,12 @@ private static int IndexOfMinMaxCore(ReadOnlySpan x) wher Vector128 remainderMask = CreateRemainderMaskSingleVector128(x.Length - i); - TIndexOfMinMax.Invoke(ref result, current, ref maxIndex, curIndex, remainderMask); + TIndexOfMinMax.Invoke(ref result, current, ref resultIndex, curIndex, remainderMask); } // Aggregate the lanes in the vector to create the final scalar result. - return TIndexOfMinMax.Invoke(result, maxIndex); + return TIndexOfMinMax.Invoke(result, resultIndex); } // Scalar path used when either vectorization is not supported or the input is too small to vectorize. diff --git a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/TensorPrimitives.netstandard.cs b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/TensorPrimitives.netstandard.cs index cde38a70fbbef4..56f9eec6014abb 100644 --- a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/TensorPrimitives.netstandard.cs +++ b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/TensorPrimitives.netstandard.cs @@ -311,6 +311,100 @@ private static float MinMaxCore(ReadOnlySpan x, TMinMaxO return result; } + /// + /// This is the same as + /// with an identity transform, except it early exits on NaN. + /// + private static int IndexOfMinMaxCore(ReadOnlySpan x, TIndexOfMinMaxOperator op = default) + where TIndexOfMinMaxOperator : struct, IIndexOfOperator + { + // This matches the IEEE 754:2019 `maximum`/`minimum` functions. + // It propagates NaN inputs back to the caller and + // otherwise returns the greater of the inputs. + // It treats +0 as greater than -0 as per the specification. + + int result; + int i; + + if (Vector.IsHardwareAccelerated && x.Length >= Vector.Count) + { + ref float xRef = ref MemoryMarshal.GetReference(x); + int[] resultIndexArray = new int[Vector.Count]; + int[] curIndexArray = new int[Vector.Count]; + int[] incrementArray = new int[Vector.Count]; + + for (int j = 0; j < Vector.Count; j++) + { + resultIndexArray[j] = j; + curIndexArray[j] = j; + incrementArray[j] = Vector.Count; + } + + + Span resultIndexSpan = new Span(resultIndexArray); + Span curIndexSpan = new Span(curIndexArray); + Span incrementSpan = new Span(incrementArray); + + + Vector resultIndex = AsVector(ref MemoryMarshal.GetReference(resultIndexSpan), 0); + Vector curIndex = AsVector(ref MemoryMarshal.GetReference(curIndexSpan), 0); + Vector increment = AsVector(ref MemoryMarshal.GetReference(incrementSpan), 0); + + // Load the first vector as the initial set of results, and bail immediately + // to scalar handling if it contains any NaNs (which don't compare equally to themselves). + Vector resultVector = AsVector(ref xRef, 0), current; + if (Vector.EqualsAll(resultVector, resultVector)) + { + int oneVectorFromEnd = x.Length - Vector.Count; + i = Vector.Count; + + // Aggregate additional vectors into the result as long as there's at least one full vector left to process. + while (i <= oneVectorFromEnd) + { + // Load the next vector, and early exit on NaN. + current = AsVector(ref xRef, i); + curIndex = Vector.Add(curIndex, increment); + + if (!Vector.EqualsAll(current, current)) + { + goto Scalar; + } + + op.Invoke(ref resultVector, current, ref resultIndex, curIndex); + i += Vector.Count; + } + + // If any elements remain, handle them in one final vector. + if (i != x.Length) + { + for(int j = 0; j < Vector.Count; j++) + { + incrementSpan[j] = x.Length - i; + } + curIndex = Vector.Add(curIndex, AsVector(ref MemoryMarshal.GetReference(incrementSpan), 0)); + + current = AsVector(ref xRef, x.Length - Vector.Count); + if (!Vector.EqualsAll(current, current)) + { + goto Scalar; + } + Vector remainderMask = CreateRemainderMaskSingleVector(x.Length - i); + + op.Invoke(ref resultVector, current, ref resultIndex, curIndex, remainderMask); + } + + result = op.Invoke(resultVector, resultIndex); + + return result; + } + } + + // Scalar path used when either vectorization is not supported, the input is too small to vectorize, + // or a NaN is encountered. + Scalar: + return op.Invoke(x); + } + /// Performs an element-wise operation on and writes the results to . /// Specifies the operation to perform on each element loaded from . private static unsafe void InvokeSpanIntoSpan( @@ -984,6 +1078,20 @@ private static ref Vector AsVector(ref float start, nuint offset) => ref Unsafe.As>( ref Unsafe.Add(ref start, (nint)(offset))); + /// Loads a that begins at the specified from . + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static ref Vector AsVector(ref int start, int offset) => + ref Unsafe.As>( + ref Unsafe.Add(ref start, offset)); + + /// Gets whether the specified is positive. + private static bool IsPositive(float f) => !IsNegative(f); + + /// Gets whether each specified is positive. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static Vector IsPositive(Vector vector) => + ((Vector)Vector.GreaterThan(((Vector)vector), Vector.Zero)); + /// Gets whether the specified is negative. private static unsafe bool IsNegative(float f) => *(int*)&f < 0; @@ -1053,6 +1161,383 @@ public Vector Invoke(Vector x, Vector y) public Vector Invoke(Vector x, Vector y) => x / y; } + private interface IIndexOfOperator + { + int Invoke(ReadOnlySpan result); + int Invoke(Vector result, Vector resultIndex); + void Invoke(ref Vector result, Vector current, ref Vector resultIndex, Vector curIndex); + void Invoke(ref Vector result, Vector current, ref Vector resultIndex, Vector curIndex, Vector remainderMask); + } + + /// MathF.Max(x, y) (but without guaranteed NaN propagation) + private readonly struct IndexOfMaxOperator : IIndexOfOperator + { + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public int Invoke(ReadOnlySpan result) + { + float curMax = result[0]; + int curIn = 0; + if (float.IsNaN(curMax)) + { + return curIn; + } + + for (int i = 1; i < result.Length; i++) + { + float current = result[i]; + if (float.IsNaN(current)) + { + return i; + } + + if (curMax == current) + { + if (IsNegative(curMax) && !IsNegative(current)) + { + curMax = current; + curIn = i; + } + } + else if (current > curMax) + { + curMax = current; + curIn = i; + } + } + + return curIn; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public int Invoke(Vector result, Vector maxIndex) + { + float curMax = float.MinValue; + int curIn = 0; + for (int i = 0; i < Vector.Count; i++) + { + if (result[i] == curMax && IsNegative(curMax) && !IsNegative(result[i])) + { + curMax = result[i]; + curIn = maxIndex[i]; + } + else if (result[i] > curMax) + { + curMax = result[i]; + curIn = maxIndex[i]; + } + } + return curIn; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public void Invoke(ref Vector max, Vector current, ref Vector maxIndex, Vector curIndex) + { + maxIndex = Vector.ConditionalSelect(Vector.Equals(max, current), + Vector.ConditionalSelect((Vector)IsNegative(max), curIndex, maxIndex), + Vector.ConditionalSelect(Vector.GreaterThan(max, current), maxIndex, curIndex)); + + max = Vector.ConditionalSelect(Vector.Equals(max, current), + Vector.ConditionalSelect(IsNegative(max), current, max), + Vector.Max(max, current)); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public void Invoke(ref Vector max, Vector current, ref Vector maxIndex, Vector curIndex, Vector remainderMask) + { + maxIndex = Vector.ConditionalSelect( + Vector.Equals((Vector)remainderMask, Vector.Zero), + maxIndex, + Vector.ConditionalSelect(Vector.Equals(max, current), + Vector.ConditionalSelect((Vector)IsNegative(max), curIndex, maxIndex), + Vector.ConditionalSelect(Vector.GreaterThan(max, current), maxIndex, curIndex))); + + max = Vector.ConditionalSelect( + Vector.Equals(remainderMask, Vector.Zero), + max, + Vector.ConditionalSelect(Vector.Equals(max, current), + Vector.ConditionalSelect(IsNegative(max), current, max), + Vector.Max(max, current))); + } + } + + private readonly struct IndexOfMaxMagnitudeOperator : IIndexOfOperator + { + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public int Invoke(ReadOnlySpan result) + { + float curMax = result[0]; + int curIn = 0; + if (float.IsNaN(curMax)) + { + return curIn; + } + + for (int i = 1; i < result.Length; i++) + { + float current = result[i]; + if (float.IsNaN(current)) + { + return i; + } + + if (MathF.Abs(curMax) == MathF.Abs(current)) + { + if (IsNegative(curMax) && !IsNegative(current)) + { + curMax = current; + curIn = i; + } + } + else if (MathF.Abs(current) > MathF.Abs(curMax)) + { + curMax = current; + curIn = i; + } + } + + return curIn; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public int Invoke(Vector result, Vector maxIndex) + { + float curMax = result[0]; + int curIn = maxIndex[0]; + for (int i = 1; i < Vector.Count; i++) + { + if (MathF.Abs(result[i]) == MathF.Abs(curMax) && IsNegative(curMax) && !IsNegative(result[i])) + { + curMax = result[i]; + curIn = maxIndex[i]; + } + else if (MathF.Abs(result[i]) > MathF.Abs(curMax)) + { + curMax = result[i]; + curIn = maxIndex[i]; + } + } + return curIn; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public void Invoke(ref Vector max, Vector current, ref Vector maxIndex, Vector curIndex) + { + Vector maxMag = Vector.Abs(max), currentMag = Vector.Abs(current); + + maxIndex = Vector.ConditionalSelect(Vector.Equals(max, current), + Vector.ConditionalSelect((Vector)IsNegative(max), curIndex, maxIndex), + Vector.ConditionalSelect(Vector.GreaterThan(maxMag, currentMag), maxIndex, curIndex)); + + max = Vector.ConditionalSelect(Vector.Equals(max, current), + Vector.ConditionalSelect(IsNegative(max), current, max), + Vector.Max(max, current)); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public void Invoke(ref Vector max, Vector current, ref Vector maxIndex, Vector curIndex, Vector remainderMask) + { + Vector maxMag = Vector.Abs(max), currentMag = Vector.Abs(current); + + maxIndex = Vector.ConditionalSelect( + Vector.Equals(((Vector)remainderMask), Vector.Zero), + maxIndex, + Vector.ConditionalSelect(Vector.Equals(max, current), + Vector.ConditionalSelect((Vector)IsNegative(max), curIndex, maxIndex), + Vector.ConditionalSelect(Vector.GreaterThan(maxMag, currentMag), maxIndex, curIndex))); + + max = Vector.ConditionalSelect( + Vector.Equals(remainderMask, Vector.Zero), + max, + Vector.ConditionalSelect(Vector.Equals(max, current), + Vector.ConditionalSelect(IsNegative(max), current, max), + Vector.Max(max, current))); + } + } + + private readonly struct IndexOfMinOperator : IIndexOfOperator + { + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public int Invoke(ReadOnlySpan result) + { + float curMin = result[0]; + int curIn = 0; + if (float.IsNaN(curMin)) + { + return curIn; + } + + for (int i = 1; i < result.Length; i++) + { + float current = result[i]; + if (float.IsNaN(current)) + { + return i; + } + + if (curMin == current) + { + if (IsPositive(curMin) && !IsPositive(current)) + { + curMin = current; + curIn = i; + } + } + else if (current < curMin) + { + curMin = current; + curIn = i; + } + } + + return curIn; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public int Invoke(Vector result, Vector resultIndex) + { + float curMin = float.MaxValue; + int curIn = 0; + for (int i = 0; i < Vector.Count; i++) + { + if (result[i] == curMin && IsPositive(curMin) && !IsPositive(result[i])) + { + curMin = result[i]; + curIn = resultIndex[i]; + } + else if (result[i] < curMin) + { + curMin = result[i]; + curIn = resultIndex[i]; + } + } + return curIn; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public void Invoke(ref Vector result, Vector current, ref Vector resultIndex, Vector curIndex) + { + resultIndex = Vector.ConditionalSelect(Vector.Equals(result, current), + Vector.ConditionalSelect(((Vector)IsPositive(result)), curIndex, resultIndex), + Vector.ConditionalSelect(Vector.LessThan(result, current), resultIndex, curIndex)); + + result = Vector.ConditionalSelect(Vector.Equals(result, current), + Vector.ConditionalSelect(IsPositive(result), current, result), + Vector.Min(result, current)); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public void Invoke(ref Vector result, Vector current, ref Vector resultIndex, Vector curIndex, Vector remainderMask) + { + resultIndex = Vector.ConditionalSelect( + Vector.Equals(((Vector)remainderMask), Vector.Zero), + resultIndex, + Vector.ConditionalSelect(Vector.Equals(result, current), + Vector.ConditionalSelect((Vector)IsPositive(result), curIndex, resultIndex), + Vector.ConditionalSelect(Vector.LessThan(result, current), resultIndex, curIndex))); + + result = Vector.ConditionalSelect( + Vector.Equals(remainderMask, Vector.Zero), + result, + Vector.ConditionalSelect(Vector.Equals(result, current), + Vector.ConditionalSelect(IsPositive(result), current, result), + Vector.Min(result, current))); + } + } + + private readonly struct IndexOfMinMagnitudeOperator : IIndexOfOperator + { + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public int Invoke(ReadOnlySpan result) + { + float curMin = result[0]; + int curIn = 0; + if (float.IsNaN(curMin)) + { + return curIn; + } + + for (int i = 1; i < result.Length; i++) + { + float current = result[i]; + if (float.IsNaN(current)) + { + return i; + } + + if (MathF.Abs(curMin) == MathF.Abs(current)) + { + if (IsPositive(curMin) && !IsPositive(current)) + { + curMin = current; + curIn = i; + } + } + else if (MathF.Abs(current) < MathF.Abs(curMin)) + { + curMin = current; + curIn = i; + } + } + + return curIn; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public int Invoke(Vector result, Vector resultIndex) + { + float curMin = result[0]; + int curIn = resultIndex[0]; + for (int i = 1; i < Vector.Count; i++) + { + if (MathF.Abs(result[i]) == MathF.Abs(curMin) && IsPositive(curMin) && !IsPositive(result[i])) + { + curMin = result[i]; + curIn = resultIndex[i]; + } + else if (MathF.Abs(result[i]) < MathF.Abs(curMin)) + { + curMin = result[i]; + curIn = resultIndex[i]; + } + } + return curIn; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public void Invoke(ref Vector result, Vector current, ref Vector resultIndex, Vector curIndex) + { + Vector minMag = Vector.Abs(result), currentMag = Vector.Abs(current); + + resultIndex = Vector.ConditionalSelect(Vector.Equals(result, current), + Vector.ConditionalSelect(((Vector)IsPositive(result)), curIndex, resultIndex), + Vector.ConditionalSelect(Vector.LessThan(minMag, currentMag), resultIndex, curIndex)); + + result = Vector.ConditionalSelect(Vector.Equals(result, current), + Vector.ConditionalSelect(IsPositive(result), current, result), + Vector.Min(minMag, currentMag)); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public void Invoke(ref Vector result, Vector current, ref Vector resultIndex, Vector curIndex, Vector remainderMask) + { + Vector minMag = Vector.Abs(result), currentMag = Vector.Abs(current); + + resultIndex = Vector.ConditionalSelect( + Vector.Equals(((Vector)remainderMask), Vector.Zero), + resultIndex, + Vector.ConditionalSelect(Vector.Equals(result, current), + Vector.ConditionalSelect(((Vector)IsPositive(result)), curIndex, resultIndex), + Vector.ConditionalSelect(Vector.LessThan(minMag, currentMag), resultIndex, curIndex))); + + result = Vector.ConditionalSelect( + Vector.Equals(remainderMask, Vector.Zero), + result, + Vector.ConditionalSelect(Vector.Equals(result, current), + Vector.ConditionalSelect(IsPositive(result), current, result), + Vector.Min(minMag, currentMag))); + } + } + /// MathF.Max(x, y) (but without guaranteed NaN propagation) private readonly struct MaxOperator : IBinaryOperator { From d53fb0c4aba31292d07e6bc8ded4765432445309 Mon Sep 17 00:00:00 2001 From: Michael Sharp Date: Fri, 13 Oct 2023 12:38:03 -0600 Subject: [PATCH 05/19] minor pr changes --- .../src/System/Numerics/Tensors/TensorPrimitives.netcore.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/TensorPrimitives.netcore.cs b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/TensorPrimitives.netcore.cs index c4a1d81d8cef3d..7981766a80eca9 100644 --- a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/TensorPrimitives.netcore.cs +++ b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/TensorPrimitives.netcore.cs @@ -1228,7 +1228,7 @@ private static int IndexOfMinMaxCore(ReadOnlySpan x) wher ref float xRef = ref MemoryMarshal.GetReference(x); Vector512 resultIndex = Vector512.Create(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); Vector512 curIndex = Vector512.Create(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); - Vector512 increment = Vector512.Create(16); + Vector512 increment = Vector512.Create(Vector512.Count); // Load the first vector as the initial set of results, and bail immediately // to scalar handling if it contains any NaNs (which don't compare equally to themselves). @@ -1284,7 +1284,7 @@ private static int IndexOfMinMaxCore(ReadOnlySpan x) wher ref float xRef = ref MemoryMarshal.GetReference(x); Vector256 resultIndex = Vector256.Create(0, 1, 2, 3, 4, 5, 6, 7); Vector256 curIndex = Vector256.Create(0, 1, 2, 3, 4, 5, 6, 7); - Vector256 increment = Vector256.Create(8); + Vector256 increment = Vector256.Create(Vector256.Count); // Load the first vector as the initial set of results, and bail immediately // to scalar handling if it contains any NaNs (which don't compare equally to themselves). @@ -1340,7 +1340,7 @@ private static int IndexOfMinMaxCore(ReadOnlySpan x) wher ref float xRef = ref MemoryMarshal.GetReference(x); Vector128 resultIndex = Vector128.Create(0, 1, 2, 3); Vector128 curIndex = Vector128.Create(0, 1, 2, 3); - Vector128 increment = Vector128.Create(4); + Vector128 increment = Vector128.Create(Vector128.Count); // Load the first vector as the initial set of results, and bail immediately // to scalar handling if it contains any NaNs (which don't compare equally to themselves). From 8f6a4d476199e9fbf0fb11c902272c5f84e803d8 Mon Sep 17 00:00:00 2001 From: Eric StJohn Date: Mon, 16 Oct 2023 13:47:06 -0700 Subject: [PATCH 06/19] Fix IndexOfMaxMagnitudeOperator --- .../System/Numerics/Tensors/TensorPrimitives.netstandard.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/TensorPrimitives.netstandard.cs b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/TensorPrimitives.netstandard.cs index 56f9eec6014abb..315e22d662a639 100644 --- a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/TensorPrimitives.netstandard.cs +++ b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/TensorPrimitives.netstandard.cs @@ -1330,7 +1330,7 @@ public void Invoke(ref Vector max, Vector current, ref Vector max = Vector.ConditionalSelect(Vector.Equals(max, current), Vector.ConditionalSelect(IsNegative(max), current, max), - Vector.Max(max, current)); + Vector.Max(maxMag, currentMag)); } [MethodImpl(MethodImplOptions.AggressiveInlining)] @@ -1350,7 +1350,7 @@ public void Invoke(ref Vector max, Vector current, ref Vector max, Vector.ConditionalSelect(Vector.Equals(max, current), Vector.ConditionalSelect(IsNegative(max), current, max), - Vector.Max(max, current))); + Vector.Max(maxMag, currentMag))); } } From b0d4c83a30969df6122b6703805110f275329ed8 Mon Sep 17 00:00:00 2001 From: Eric StJohn Date: Mon, 16 Oct 2023 16:13:01 -0700 Subject: [PATCH 07/19] Fix IndexOfMaxMagnitudeOperator on netcore --- .../System/Numerics/Tensors/TensorPrimitives.netcore.cs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/TensorPrimitives.netcore.cs b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/TensorPrimitives.netcore.cs index 7981766a80eca9..d0ab3d60cc2690 100644 --- a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/TensorPrimitives.netcore.cs +++ b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/TensorPrimitives.netcore.cs @@ -3603,7 +3603,7 @@ public static void Invoke(ref Vector128 max, Vector128 current, re max = Vector128.ConditionalSelect(Vector128.Equals(max, current), Vector128.ConditionalSelect(IsNegative(max), current, max), - Vector128.Max(max, current)); + Vector128.Max(maxMag, currentMag)); } [MethodImpl(MethodImplOptions.AggressiveInlining)] @@ -3621,7 +3621,7 @@ public static void Invoke(ref Vector128 max, Vector128 current, re max = Vector128.ConditionalSelect( Vector128.Equals(remainderMask, Vector128.Zero), max, - MaxOperator.Invoke(max, current)); + MaxOperator.Invoke(maxMag, currentMag)); } [MethodImpl(MethodImplOptions.AggressiveInlining)] @@ -3656,7 +3656,7 @@ public static void Invoke(ref Vector256 max, Vector256 current, re max = Vector256.ConditionalSelect(Vector256.Equals(max, current), Vector256.ConditionalSelect(IsNegative(max), current, max), - Vector256.Max(max, current)); + Vector256.Max(maxMag, currentMag)); } [MethodImpl(MethodImplOptions.AggressiveInlining)] @@ -3674,7 +3674,7 @@ public static void Invoke(ref Vector256 max, Vector256 current, re max = Vector256.ConditionalSelect( Vector256.Equals(remainderMask, Vector256.Zero), max, - MaxOperator.Invoke(max, current)); + MaxOperator.Invoke(maxMag, currentMag)); } #if NET8_0_OR_GREATER From 2c1fbc9c431bee7878f67b640884947d8ceb6915 Mon Sep 17 00:00:00 2001 From: Michael Sharp Date: Tue, 17 Oct 2023 03:08:46 -0600 Subject: [PATCH 08/19] updates from PR comments --- .../Tensors/TensorPrimitives.netcore.cs | 99 ++++++++++++------- 1 file changed, 66 insertions(+), 33 deletions(-) diff --git a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/TensorPrimitives.netcore.cs b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/TensorPrimitives.netcore.cs index d0ab3d60cc2690..0763920506e1fd 100644 --- a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/TensorPrimitives.netcore.cs +++ b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/TensorPrimitives.netcore.cs @@ -1232,10 +1232,13 @@ private static int IndexOfMinMaxCore(ReadOnlySpan x) wher // Load the first vector as the initial set of results, and bail immediately // to scalar handling if it contains any NaNs (which don't compare equally to themselves). - Vector512 result = Vector512.LoadUnsafe(ref xRef, 0), current; - if (!Vector512.EqualsAll(result, result)) + Vector512 result = Vector512.LoadUnsafe(ref xRef); + Vector512 current; + Vector512 nanMask = ~Vector512.Equals(result, result); + + if (nanMask != Vector512.Zero) { - return GetFirstNaNIndex(result, resultIndex); + return IndexOfFirstMatch(nanMask); } int oneVectorFromEnd = x.Length - Vector512.Count; @@ -1246,11 +1249,13 @@ private static int IndexOfMinMaxCore(ReadOnlySpan x) wher { // Load the next vector, and early exit on NaN. current = Vector512.LoadUnsafe(ref xRef, (uint)i); - curIndex = Vector512.Add(curIndex, increment); + curIndex += increment; - if (!Vector512.EqualsAll(current, current)) + nanMask = ~Vector512.Equals(current, current); + + if (nanMask != Vector512.Zero) { - return GetFirstNaNIndex(current, curIndex); + return i + IndexOfFirstMatch(nanMask); } TIndexOfMinMax.Invoke(ref result, current, ref resultIndex, curIndex); @@ -1261,11 +1266,13 @@ private static int IndexOfMinMaxCore(ReadOnlySpan x) wher if (i != x.Length) { current = Vector512.LoadUnsafe(ref xRef, (uint)(x.Length - Vector512.Count)); - curIndex = Vector512.Add(curIndex, Vector512.Create(x.Length - i)); + curIndex += Vector512.Create(x.Length - i); - if (!Vector512.EqualsAll(current, current)) + nanMask = ~Vector512.Equals(current, current); + + if (nanMask != Vector512.Zero) { - return GetFirstNaNIndex(current, curIndex); + return curIndex[IndexOfFirstMatch(nanMask)]; } Vector512 remainderMask = CreateRemainderMaskSingleVector512(x.Length - i); @@ -1288,10 +1295,13 @@ private static int IndexOfMinMaxCore(ReadOnlySpan x) wher // Load the first vector as the initial set of results, and bail immediately // to scalar handling if it contains any NaNs (which don't compare equally to themselves). - Vector256 max = Vector256.LoadUnsafe(ref xRef, 0), current; - if (!Vector256.EqualsAll(max, max)) + Vector256 result = Vector256.LoadUnsafe(ref xRef); + Vector256 current; + Vector256 nanMask = ~Vector256.Equals(result, result); + + if (nanMask != Vector256.Zero) { - return GetFirstNaNIndex(max, resultIndex); + return IndexOfFirstMatch(nanMask); } int oneVectorFromEnd = x.Length - Vector256.Count; @@ -1302,14 +1312,16 @@ private static int IndexOfMinMaxCore(ReadOnlySpan x) wher { // Load the next vector, and early exit on NaN. current = Vector256.LoadUnsafe(ref xRef, (uint)i); - curIndex = Vector256.Add(curIndex, increment); + curIndex += increment; - if (!Vector256.EqualsAll(current, current)) + nanMask = ~Vector256.Equals(current, current); + + if (nanMask != Vector256.Zero) { - return GetFirstNaNIndex(current, curIndex); + return i + IndexOfFirstMatch(nanMask); } - TIndexOfMinMax.Invoke(ref max, current, ref resultIndex, curIndex); + TIndexOfMinMax.Invoke(ref result, current, ref resultIndex, curIndex); i += Vector256.Count; } @@ -1318,21 +1330,23 @@ private static int IndexOfMinMaxCore(ReadOnlySpan x) wher if (i != x.Length) { current = Vector256.LoadUnsafe(ref xRef, (uint)(x.Length - Vector256.Count)); - curIndex = Vector256.Add(curIndex, Vector256.Create(x.Length - i)); + curIndex += Vector256.Create(x.Length - i); - if (!Vector256.EqualsAll(current, current)) + nanMask = ~Vector256.Equals(current, current); + + if (nanMask != Vector256.Zero) { - return GetFirstNaNIndex(current, curIndex); + return curIndex[IndexOfFirstMatch(nanMask)]; } Vector256 remainderMask = CreateRemainderMaskSingleVector256(x.Length - i); - TIndexOfMinMax.Invoke(ref max, current, ref resultIndex, curIndex, remainderMask); + TIndexOfMinMax.Invoke(ref result, current, ref resultIndex, curIndex, remainderMask); } // Aggregate the lanes in the vector to create the final scalar result. - return TIndexOfMinMax.Invoke(max, resultIndex); + return TIndexOfMinMax.Invoke(result, resultIndex); } if (Vector128.IsHardwareAccelerated && x.Length >= Vector128.Count) @@ -1344,10 +1358,13 @@ private static int IndexOfMinMaxCore(ReadOnlySpan x) wher // Load the first vector as the initial set of results, and bail immediately // to scalar handling if it contains any NaNs (which don't compare equally to themselves). - Vector128 result = Vector128.LoadUnsafe(ref xRef, 0), current; - if (!Vector128.EqualsAll(result, result)) + Vector128 result = Vector128.LoadUnsafe(ref xRef); + Vector128 current; + Vector128 nanMask = ~Vector128.Equals(result, result); + + if (nanMask != Vector128.Zero) { - return GetFirstNaNIndex(result, resultIndex); + return IndexOfFirstMatch(nanMask); } int oneVectorFromEnd = x.Length - Vector128.Count; @@ -1358,11 +1375,13 @@ private static int IndexOfMinMaxCore(ReadOnlySpan x) wher { // Load the next vector, and early exit on NaN. current = Vector128.LoadUnsafe(ref xRef, (uint)i); - curIndex = Vector128.Add(curIndex, increment); + curIndex += increment; - if (!Vector128.EqualsAll(current, current)) + nanMask = ~Vector128.Equals(current, current); + + if (nanMask != Vector128.Zero) { - return GetFirstNaNIndex(current, curIndex); + return i + IndexOfFirstMatch(nanMask); } TIndexOfMinMax.Invoke(ref result, current, ref resultIndex, curIndex); @@ -1373,12 +1392,14 @@ private static int IndexOfMinMaxCore(ReadOnlySpan x) wher // If any elements remain, handle them in one final vector. if (i != x.Length) { - curIndex = Vector128.Add(curIndex, Vector128.Create(x.Length - i)); + curIndex += Vector128.Create(x.Length - i); current = Vector128.LoadUnsafe(ref xRef, (uint)(x.Length - Vector128.Count)); - if (!Vector128.EqualsAll(current, current)) + nanMask = ~Vector128.Equals(current, current); + + if (nanMask != Vector128.Zero) { - return GetFirstNaNIndex(current, curIndex); + return curIndex[IndexOfFirstMatch(nanMask)]; } Vector128 remainderMask = CreateRemainderMaskSingleVector128(x.Length - i); @@ -1392,11 +1413,23 @@ private static int IndexOfMinMaxCore(ReadOnlySpan x) wher } // Scalar path used when either vectorization is not supported or the input is too small to vectorize. - { - return TIndexOfMinMax.Invoke(x); - } + return TIndexOfMinMax.Invoke(x); } +#if NET8_0_OR_GREATER + private static int IndexOfFirstMatch(Vector512 mask) + { + return BitOperations.TrailingZeroCount(mask.ExtractMostSignificantBits()); + } +#endif + private static int IndexOfFirstMatch(Vector256 mask) + { + return BitOperations.TrailingZeroCount(mask.ExtractMostSignificantBits()); + } + private static int IndexOfFirstMatch(Vector128 mask) + { + return BitOperations.TrailingZeroCount(mask.ExtractMostSignificantBits()); + } /// Performs an element-wise operation on and writes the results to . /// Specifies the operation to perform on each element loaded from . private static void InvokeSpanIntoSpan( From b1a9efc1321428ede7201eedc8a2a0f922d14526 Mon Sep 17 00:00:00 2001 From: Michael Sharp Date: Wed, 18 Oct 2023 13:09:10 -0600 Subject: [PATCH 09/19] netcore fixed --- .../Tensors/TensorPrimitives.netcore.cs | 787 ++++++++---------- 1 file changed, 337 insertions(+), 450 deletions(-) diff --git a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/TensorPrimitives.netcore.cs b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/TensorPrimitives.netcore.cs index 0763920506e1fd..315e1514e7b3ed 100644 --- a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/TensorPrimitives.netcore.cs +++ b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/TensorPrimitives.netcore.cs @@ -1275,9 +1275,7 @@ private static int IndexOfMinMaxCore(ReadOnlySpan x) wher return curIndex[IndexOfFirstMatch(nanMask)]; } - Vector512 remainderMask = CreateRemainderMaskSingleVector512(x.Length - i); - - TIndexOfMinMax.Invoke(ref result, current, ref resultIndex, curIndex, remainderMask); + TIndexOfMinMax.Invoke(ref result, current, ref resultIndex, curIndex); } // Aggregate the lanes in the vector to create the final scalar result. @@ -1339,9 +1337,7 @@ private static int IndexOfMinMaxCore(ReadOnlySpan x) wher return curIndex[IndexOfFirstMatch(nanMask)]; } - Vector256 remainderMask = CreateRemainderMaskSingleVector256(x.Length - i); - - TIndexOfMinMax.Invoke(ref result, current, ref resultIndex, curIndex, remainderMask); + TIndexOfMinMax.Invoke(ref result, current, ref resultIndex, curIndex); } @@ -1402,9 +1398,7 @@ private static int IndexOfMinMaxCore(ReadOnlySpan x) wher return curIndex[IndexOfFirstMatch(nanMask)]; } - Vector128 remainderMask = CreateRemainderMaskSingleVector128(x.Length - i); - - TIndexOfMinMax.Invoke(ref result, current, ref resultIndex, curIndex, remainderMask); + TIndexOfMinMax.Invoke(ref result, current, ref resultIndex, curIndex); } @@ -3364,14 +3358,11 @@ private interface IIndexOfOperator static abstract int Invoke(ReadOnlySpan result); static abstract int Invoke(Vector128 result, Vector128 resultIndex); static abstract void Invoke(ref Vector128 result, Vector128 current, ref Vector128 resultIndex, Vector128 curIndex); - static abstract void Invoke(ref Vector128 result, Vector128 current, ref Vector128 resultIndex, Vector128 curIndex, Vector128 remainderMask); static abstract int Invoke(Vector256 result, Vector256 resultIndex); static abstract void Invoke(ref Vector256 result, Vector256 current, ref Vector256 resultIndex, Vector256 curIndex); - static abstract void Invoke(ref Vector256 result, Vector256 current, ref Vector256 resultIndex, Vector256 curIndex, Vector256 remainderMask); #if NET8_0_OR_GREATER static abstract int Invoke(Vector512 result, Vector512 resultIndex); static abstract void Invoke(ref Vector512 result, Vector512 current, ref Vector512 resultIndex, Vector512 curIndex); - static abstract void Invoke(ref Vector512 result, Vector512 current, ref Vector512 resultIndex, Vector512 curIndex, Vector512 remainderMask); #endif static virtual float IdentityValue => throw new NotSupportedException(); @@ -3419,149 +3410,112 @@ public static int Invoke(ReadOnlySpan result) [MethodImpl(MethodImplOptions.AggressiveInlining)] public static int Invoke(Vector128 result, Vector128 maxIndex) { - float curMax = float.MinValue; - int curIn = 0; - for (int i = 0; i < Vector128.Count; i++) - { - if (result[i] == curMax && IsNegative(curMax) && !IsNegative(result[i])) - { - curMax = result[i]; - curIn = maxIndex[i]; - } - else if (result[i] > curMax) - { - curMax = result[i]; - curIn = maxIndex[i]; - } - } - return curIn; + Vector128 tmp128 = Vector128.Shuffle(result, Vector128.Create(2, 3, 0, 1)); + Vector128 tmpIndex128 = Vector128.Shuffle(maxIndex, Vector128.Create(2, 3, 0, 1)); + + Invoke(ref result, tmp128, ref maxIndex, tmpIndex128); + + tmp128 = Vector128.Shuffle(result, Vector128.Create(1, 0, 3, 2)); + tmpIndex128 = Vector128.Shuffle(maxIndex, Vector128.Create(1, 0, 3, 2)); + + Invoke(ref result, tmp128, ref maxIndex, tmpIndex128); + return maxIndex.ToScalar(); } [MethodImpl(MethodImplOptions.AggressiveInlining)] public static void Invoke(ref Vector128 max, Vector128 current, ref Vector128 maxIndex, Vector128 curIndex) { - maxIndex = Vector128.ConditionalSelect(Vector128.Equals(max, current).AsInt32(), - Vector128.ConditionalSelect(IsNegative(max).AsInt32(), curIndex, maxIndex), - Vector128.ConditionalSelect(Vector128.GreaterThan(max, current).AsInt32(), maxIndex, curIndex)); + Vector128 greaterThanMask = Vector128.GreaterThan(max, current); - max = Vector128.ConditionalSelect(Vector128.Equals(max, current), - Vector128.ConditionalSelect(IsNegative(max), current, max), - Vector128.Max(max, current)); - } + Vector128 equalMask = Vector128.Equals(max, current); - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static void Invoke(ref Vector128 max, Vector128 current, ref Vector128 maxIndex, Vector128 curIndex, Vector128 remainderMask) - { - maxIndex = Vector128.ConditionalSelect( - Vector128.Equals(remainderMask.AsInt32(), Vector128.Zero), - maxIndex, - Vector128.ConditionalSelect(Vector128.Equals(max, current).AsInt32(), - Vector128.ConditionalSelect(IsNegative(max).AsInt32(), curIndex, maxIndex), - Vector128.ConditionalSelect(Vector128.GreaterThan(max, current).AsInt32(), maxIndex, curIndex))); + if (equalMask.AsInt32() != Vector128.Zero) + { + Vector128 negativeMask = IsNegative(current); + Vector128 lessThanMask = Vector128.LessThan(maxIndex, curIndex); + + greaterThanMask |= (negativeMask & equalMask) | (~IsNegative(max) & equalMask & lessThanMask.AsSingle()); + } - max = Vector128.ConditionalSelect( - Vector128.Equals(remainderMask, Vector128.Zero), - max, - MaxOperator.Invoke(max, current)); + greaterThanMask = ~greaterThanMask; + + max = ElementWiseSelect(greaterThanMask, max, current); + + maxIndex = ElementWiseSelect(greaterThanMask.AsInt32(), maxIndex, curIndex); } [MethodImpl(MethodImplOptions.AggressiveInlining)] public static int Invoke(Vector256 result, Vector256 maxIndex) { - float curMax = float.MinValue; - int curIn = 0; - for (int i = 0; i < Vector256.Count; i++) - { - if (result[i] == curMax && IsNegative(curMax) && !IsNegative(result[i])) - { - curMax = result[i]; - curIn = maxIndex[i]; - } - else if (result[i] > curMax) - { - curMax = result[i]; - curIn = maxIndex[i]; - } - } - return curIn; + // Max the upper/lower halves of the Vector256 + Vector128 resultLower = result.GetLower(); + Vector128 indexLower = maxIndex.GetLower(); + + Invoke(ref resultLower, result.GetUpper(), ref indexLower, maxIndex.GetUpper()); + return Invoke(resultLower, indexLower); } [MethodImpl(MethodImplOptions.AggressiveInlining)] public static void Invoke(ref Vector256 max, Vector256 current, ref Vector256 maxIndex, Vector256 curIndex) { - maxIndex = Vector256.ConditionalSelect(Vector256.Equals(max, current).AsInt32(), - Vector256.ConditionalSelect(IsNegative(max).AsInt32(), curIndex, maxIndex), - Vector256.ConditionalSelect(Vector256.GreaterThan(max, current).AsInt32(), maxIndex, curIndex)); + Vector256 greaterThanMask = Vector256.GreaterThan(max, current); - max = Vector256.ConditionalSelect(Vector256.Equals(max, current), - Vector256.ConditionalSelect(IsNegative(max), current, max), - Vector256.Max(max, current)); - } + Vector256 equalMask = Vector256.Equals(max, current); - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static void Invoke(ref Vector256 max, Vector256 current, ref Vector256 maxIndex, Vector256 curIndex, Vector256 remainderMask) - { - maxIndex = Vector256.ConditionalSelect( - Vector256.Equals(remainderMask.AsInt32(), Vector256.Zero), - maxIndex, - Vector256.ConditionalSelect(Vector256.Equals(max, current).AsInt32(), - Vector256.ConditionalSelect(IsNegative(max).AsInt32(), curIndex, maxIndex), - Vector256.ConditionalSelect(Vector256.GreaterThan(max, current).AsInt32(), maxIndex, curIndex))); + if (equalMask.AsInt32() != Vector256.Zero) + { + Vector256 negativeMask = IsNegative(current); + Vector256 lessThanMask = Vector256.LessThan(maxIndex, curIndex); + + greaterThanMask |= (negativeMask & equalMask) | (~IsNegative(max) & equalMask & lessThanMask.AsSingle()); + } - max = Vector256.ConditionalSelect( - Vector256.Equals(remainderMask, Vector256.Zero), - max, - MaxOperator.Invoke(max, current)); + greaterThanMask = ~greaterThanMask; + + max = ElementWiseSelect(greaterThanMask, max, current); + + maxIndex = ElementWiseSelect(greaterThanMask.AsInt32(), maxIndex, curIndex); } #if NET8_0_OR_GREATER [MethodImpl(MethodImplOptions.AggressiveInlining)] public static int Invoke(Vector512 result, Vector512 maxIndex) { - float curMax = float.MinValue; - int curIn = 0; - for (int i = 0; i < Vector512.Count; i++) - { - if (result[i] == curMax && IsNegative(curMax) && !IsNegative(result[i])) - { - curMax = result[i]; - curIn = maxIndex[i]; - } - else if (result[i] > curMax) - { - curMax = result[i]; - curIn = maxIndex[i]; - } - } - return curIn; + // Max the upper/lower halves of the Vector512 + Vector256 resultLower256 = result.GetLower(); + Vector256 indexLower256 = maxIndex.GetLower(); + + Invoke(ref resultLower256, result.GetUpper(), ref indexLower256, maxIndex.GetUpper()); + + // Max the upper/lower halves of the Vector256 + Vector128 resultLower128 = resultLower256.GetLower(); + Vector128 indexLower128 = indexLower256.GetLower(); + + Invoke(ref resultLower128, resultLower256.GetUpper(), ref indexLower128, indexLower256.GetUpper()); + + return Invoke(resultLower128, indexLower128); } [MethodImpl(MethodImplOptions.AggressiveInlining)] public static void Invoke(ref Vector512 max, Vector512 current, ref Vector512 maxIndex, Vector512 curIndex) { - maxIndex = Vector512.ConditionalSelect(Vector512.Equals(max, current).AsInt32(), - Vector512.ConditionalSelect(IsNegative(max).AsInt32(), curIndex, maxIndex), - Vector512.ConditionalSelect(Vector512.GreaterThan(max, current).AsInt32(), maxIndex, curIndex)); + Vector512 greaterThanMask = Vector512.GreaterThan(max, current); - max = Vector512.ConditionalSelect(Vector512.Equals(max, current), - Vector512.ConditionalSelect(IsNegative(max), current, max), - Vector512.Max(max, current)); - } + Vector512 equalMask = Vector512.Equals(max, current); - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static void Invoke(ref Vector512 max, Vector512 current, ref Vector512 maxIndex, Vector512 curIndex, Vector512 remainderMask) - { - maxIndex = Vector512.ConditionalSelect( - Vector512.Equals(remainderMask.AsInt32(), Vector512.Zero), - maxIndex, - Vector512.ConditionalSelect(Vector512.Equals(max, current).AsInt32(), - Vector512.ConditionalSelect(IsNegative(max).AsInt32(), curIndex, maxIndex), - Vector512.ConditionalSelect(Vector512.GreaterThan(max, current).AsInt32(), maxIndex, curIndex))); + if (equalMask.AsInt32() != Vector512.Zero) + { + Vector512 negativeMask = IsNegative(current); + Vector512 lessThanMask = Vector512.LessThan(maxIndex, curIndex); + + greaterThanMask |= (negativeMask & equalMask) | (~IsNegative(max) & equalMask & lessThanMask.AsSingle()); + } - max = Vector512.ConditionalSelect( - Vector512.Equals(remainderMask, Vector512.Zero), - max, - MaxOperator.Invoke(max, current)); + greaterThanMask = ~greaterThanMask; + + max = ElementWiseSelect(greaterThanMask, max, current); + + maxIndex = ElementWiseSelect(greaterThanMask.AsInt32(), maxIndex, curIndex); } #endif } @@ -3607,22 +3561,16 @@ public static int Invoke(ReadOnlySpan result) [MethodImpl(MethodImplOptions.AggressiveInlining)] public static int Invoke(Vector128 result, Vector128 maxIndex) { - float curMax = result[0]; - int curIn = maxIndex[0]; - for (int i = 1; i < Vector128.Count; i++) - { - if (MathF.Abs(result[i]) == MathF.Abs(curMax) && IsNegative(curMax) && !IsNegative(result[i])) - { - curMax = result[i]; - curIn = maxIndex[i]; - } - else if (MathF.Abs(result[i]) > MathF.Abs(curMax)) - { - curMax = result[i]; - curIn = maxIndex[i]; - } - } - return curIn; + Vector128 tmp128 = Vector128.Shuffle(result, Vector128.Create(2, 3, 0, 1)); + Vector128 tmpIndex128 = Vector128.Shuffle(maxIndex, Vector128.Create(2, 3, 0, 1)); + + Invoke(ref result, tmp128, ref maxIndex, tmpIndex128); + + tmp128 = Vector128.Shuffle(result, Vector128.Create(1, 0, 3, 2)); + tmpIndex128 = Vector128.Shuffle(maxIndex, Vector128.Create(1, 0, 3, 2)); + + Invoke(ref result, tmp128, ref maxIndex, tmpIndex128); + return maxIndex.ToScalar(); } [MethodImpl(MethodImplOptions.AggressiveInlining)] @@ -3630,52 +3578,34 @@ public static void Invoke(ref Vector128 max, Vector128 current, re { Vector128 maxMag = Vector128.Abs(max), currentMag = Vector128.Abs(current); - maxIndex = Vector128.ConditionalSelect(Vector128.Equals(max, current).AsInt32(), - Vector128.ConditionalSelect(IsNegative(max).AsInt32(), curIndex, maxIndex), - Vector128.ConditionalSelect(Vector128.GreaterThan(maxMag, currentMag).AsInt32(), maxIndex, curIndex)); + Vector128 greaterThanMask = Vector128.GreaterThan(maxMag, currentMag); - max = Vector128.ConditionalSelect(Vector128.Equals(max, current), - Vector128.ConditionalSelect(IsNegative(max), current, max), - Vector128.Max(maxMag, currentMag)); - } + Vector128 equalMask = Vector128.Equals(max, current); - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static void Invoke(ref Vector128 max, Vector128 current, ref Vector128 maxIndex, Vector128 curIndex, Vector128 remainderMask) - { - Vector128 maxMag = Vector128.Abs(max), currentMag = Vector128.Abs(current); + if (equalMask.AsInt32() != Vector128.Zero) + { + Vector128 negativeMask = IsNegative(current); + Vector128 lessThanMask = Vector128.LessThan(maxIndex, curIndex); + + greaterThanMask |= (negativeMask & equalMask) | (~IsNegative(max) & equalMask & lessThanMask.AsSingle()); + } + + greaterThanMask = ~greaterThanMask; - maxIndex = Vector128.ConditionalSelect( - Vector128.Equals(remainderMask.AsInt32(), Vector128.Zero), - maxIndex, - Vector128.ConditionalSelect(Vector128.Equals(max, current).AsInt32(), - Vector128.ConditionalSelect(IsNegative(max).AsInt32(), curIndex, maxIndex), - Vector128.ConditionalSelect(Vector128.GreaterThan(maxMag, currentMag).AsInt32(), maxIndex, curIndex))); + max = ElementWiseSelect(greaterThanMask, max, current); - max = Vector128.ConditionalSelect( - Vector128.Equals(remainderMask, Vector128.Zero), - max, - MaxOperator.Invoke(maxMag, currentMag)); + maxIndex = ElementWiseSelect(greaterThanMask.AsInt32(), maxIndex, curIndex); } [MethodImpl(MethodImplOptions.AggressiveInlining)] public static int Invoke(Vector256 result, Vector256 maxIndex) { - float curMax = result[0]; - int curIn = maxIndex[0]; - for (int i = 1; i < Vector256.Count; i++) - { - if (MathF.Abs(result[i]) == MathF.Abs(curMax) && IsNegative(curMax) && !IsNegative(result[i])) - { - curMax = result[i]; - curIn = maxIndex[i]; - } - else if (MathF.Abs(result[i]) > MathF.Abs(curMax)) - { - curMax = result[i]; - curIn = maxIndex[i]; - } - } - return curIn; + // Max the upper/lower halves of the Vector256 + Vector128 resultLower = result.GetLower(); + Vector128 indexLower = maxIndex.GetLower(); + + Invoke(ref resultLower, result.GetUpper(), ref indexLower, maxIndex.GetUpper()); + return Invoke(resultLower, indexLower); } [MethodImpl(MethodImplOptions.AggressiveInlining)] @@ -3683,85 +3613,65 @@ public static void Invoke(ref Vector256 max, Vector256 current, re { Vector256 maxMag = Vector256.Abs(max), currentMag = Vector256.Abs(current); - maxIndex = Vector256.ConditionalSelect(Vector256.Equals(max, current).AsInt32(), - Vector256.ConditionalSelect(IsNegative(max).AsInt32(), curIndex, maxIndex), - Vector256.ConditionalSelect(Vector256.GreaterThan(maxMag, currentMag).AsInt32(), maxIndex, curIndex)); + Vector256 greaterThanMask = Vector256.GreaterThan(maxMag, currentMag); - max = Vector256.ConditionalSelect(Vector256.Equals(max, current), - Vector256.ConditionalSelect(IsNegative(max), current, max), - Vector256.Max(maxMag, currentMag)); - } + Vector256 equalMask = Vector256.Equals(max, current); - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static void Invoke(ref Vector256 max, Vector256 current, ref Vector256 maxIndex, Vector256 curIndex, Vector256 remainderMask) - { - Vector256 maxMag = Vector256.Abs(max), currentMag = Vector256.Abs(current); + if (equalMask.AsInt32() != Vector256.Zero) + { + Vector256 negativeMask = IsNegative(current); + Vector256 lessThanMask = Vector256.LessThan(maxIndex, curIndex); + + greaterThanMask |= (negativeMask & equalMask) | (~IsNegative(max) & equalMask & lessThanMask.AsSingle()); + } + + greaterThanMask = ~greaterThanMask; - maxIndex = Vector256.ConditionalSelect( - Vector256.Equals(remainderMask.AsInt32(), Vector256.Zero), - maxIndex, - Vector256.ConditionalSelect(Vector256.Equals(max, current).AsInt32(), - Vector256.ConditionalSelect(IsNegative(max).AsInt32(), curIndex, maxIndex), - Vector256.ConditionalSelect(Vector256.GreaterThan(maxMag, currentMag).AsInt32(), maxIndex, curIndex))); + max = ElementWiseSelect(greaterThanMask, max, current); - max = Vector256.ConditionalSelect( - Vector256.Equals(remainderMask, Vector256.Zero), - max, - MaxOperator.Invoke(maxMag, currentMag)); + maxIndex = ElementWiseSelect(greaterThanMask.AsInt32(), maxIndex, curIndex); } #if NET8_0_OR_GREATER [MethodImpl(MethodImplOptions.AggressiveInlining)] public static int Invoke(Vector512 result, Vector512 maxIndex) { - float curMax = result[0]; - int curIn = maxIndex[0]; - for (int i = 1; i < Vector512.Count; i++) - { - if (MathF.Abs(result[i]) == MathF.Abs(curMax) && IsNegative(curMax) && !IsNegative(result[i])) - { - curMax = result[i]; - curIn = maxIndex[i]; - } - else if (MathF.Abs(result[i]) > MathF.Abs(curMax)) - { - curMax = result[i]; - curIn = maxIndex[i]; - } - } - return curIn; + // Max the upper/lower halves of the Vector512 + Vector256 resultLower256 = result.GetLower(); + Vector256 indexLower256 = maxIndex.GetLower(); + + Invoke(ref resultLower256, result.GetUpper(), ref indexLower256, maxIndex.GetUpper()); + + // Max the upper/lower halves of the Vector256 + Vector128 resultLower128 = resultLower256.GetLower(); + Vector128 indexLower128 = indexLower256.GetLower(); + + Invoke(ref resultLower128, resultLower256.GetUpper(), ref indexLower128, indexLower256.GetUpper()); + + return Invoke(resultLower128, indexLower128); } [MethodImpl(MethodImplOptions.AggressiveInlining)] public static void Invoke(ref Vector512 max, Vector512 current, ref Vector512 maxIndex, Vector512 curIndex) { Vector512 maxMag = Vector512.Abs(max), currentMag = Vector512.Abs(current); + Vector512 greaterThanMask = Vector512.GreaterThan(maxMag, currentMag); - maxIndex = Vector512.ConditionalSelect(Vector512.Equals(max, current).AsInt32(), - Vector512.ConditionalSelect(IsNegative(max).AsInt32(), curIndex, maxIndex), - Vector512.ConditionalSelect(Vector512.GreaterThan(maxMag, currentMag).AsInt32(), maxIndex, curIndex)); + Vector512 equalMask = Vector512.Equals(max, current); - max = Vector512.ConditionalSelect(Vector512.Equals(max, current), - Vector512.ConditionalSelect(IsNegative(max), current, max), - Vector512.Max(maxMag, currentMag)); - } + if (equalMask.AsInt32() != Vector512.Zero) + { + Vector512 negativeMask = IsNegative(current); + Vector512 lessThanMask = Vector512.LessThan(maxIndex, curIndex); - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static void Invoke(ref Vector512 max, Vector512 current, ref Vector512 maxIndex, Vector512 curIndex, Vector512 remainderMask) - { - Vector512 maxMag = Vector512.Abs(max), currentMag = Vector512.Abs(current); + greaterThanMask |= (negativeMask & equalMask) | (~IsNegative(max) & equalMask & lessThanMask.AsSingle()); + } + + greaterThanMask = ~greaterThanMask; - maxIndex = Vector512.ConditionalSelect( - Vector512.Equals(remainderMask.AsInt32(), Vector512.Zero), - maxIndex, - Vector512.ConditionalSelect(Vector512.Equals(max, current).AsInt32(), - Vector512.ConditionalSelect(IsNegative(max).AsInt32(), curIndex, maxIndex), - Vector512.ConditionalSelect(Vector512.GreaterThan(maxMag, currentMag).AsInt32(), maxIndex, curIndex))); + max = ElementWiseSelect(greaterThanMask, max, current); - max = Vector512.ConditionalSelect( - Vector512.Equals(remainderMask, Vector512.Zero), - max, - MaxOperator.Invoke(maxMag, currentMag)); + maxIndex = ElementWiseSelect(greaterThanMask.AsInt32(), maxIndex, curIndex); } #endif } @@ -3808,149 +3718,113 @@ public static int Invoke(ReadOnlySpan result) [MethodImpl(MethodImplOptions.AggressiveInlining)] public static int Invoke(Vector128 result, Vector128 resultIndex) { - float curMin = float.MaxValue; - int curIn = 0; - for (int i = 0; i < Vector128.Count; i++) - { - if (result[i] == curMin && IsPositive(curMin) && !IsPositive(result[i])) - { - curMin = result[i]; - curIn = resultIndex[i]; - } - else if (result[i] < curMin) - { - curMin = result[i]; - curIn = resultIndex[i]; - } - } - return curIn; + Vector128 tmp128 = Vector128.Shuffle(result, Vector128.Create(2, 3, 0, 1)); + Vector128 tmpIndex128 = Vector128.Shuffle(resultIndex, Vector128.Create(2, 3, 0, 1)); + + Invoke(ref result, tmp128, ref resultIndex, tmpIndex128); + + tmp128 = Vector128.Shuffle(result, Vector128.Create(1, 0, 3, 2)); + tmpIndex128 = Vector128.Shuffle(resultIndex, Vector128.Create(1, 0, 3, 2)); + + Invoke(ref result, tmp128, ref resultIndex, tmpIndex128); + return resultIndex.ToScalar(); } [MethodImpl(MethodImplOptions.AggressiveInlining)] public static void Invoke(ref Vector128 result, Vector128 current, ref Vector128 resultIndex, Vector128 curIndex) { - resultIndex = Vector128.ConditionalSelect(Vector128.Equals(result, current).AsInt32(), - Vector128.ConditionalSelect(IsPositive(result).AsInt32(), curIndex, resultIndex), - Vector128.ConditionalSelect(Vector128.LessThan(result, current).AsInt32(), resultIndex, curIndex)); + Vector128 lessThanMask = Vector128.LessThan(result, current); - result = Vector128.ConditionalSelect(Vector128.Equals(result, current), - Vector128.ConditionalSelect(IsPositive(result), current, result), - Vector128.Min(result, current)); - } + Vector128 equalMask = Vector128.Equals(result, current); - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static void Invoke(ref Vector128 result, Vector128 current, ref Vector128 resultIndex, Vector128 curIndex, Vector128 remainderMask) - { - resultIndex = Vector128.ConditionalSelect( - Vector128.Equals(remainderMask.AsInt32(), Vector128.Zero), - resultIndex, - Vector128.ConditionalSelect(Vector128.Equals(result, current).AsInt32(), - Vector128.ConditionalSelect(IsPositive(result).AsInt32(), curIndex, resultIndex), - Vector128.ConditionalSelect(Vector128.LessThan(result, current).AsInt32(), resultIndex, curIndex))); + if (equalMask.AsInt32() != Vector128.Zero) + { + Vector128 negativeMask = IsNegative(current); + Vector128 lessThanIndexMask = Vector128.LessThan(resultIndex, curIndex); + + lessThanMask |= (~negativeMask & equalMask) | (IsNegative(result) & equalMask & lessThanIndexMask.AsSingle()); + } + + lessThanMask = ~lessThanMask; + + result = ElementWiseSelect(lessThanMask, result, current); - result = Vector128.ConditionalSelect( - Vector128.Equals(remainderMask, Vector128.Zero), - result, - MinOperator.Invoke(result, current)); + resultIndex = ElementWiseSelect(lessThanMask.AsInt32(), resultIndex, curIndex); } + [MethodImpl(MethodImplOptions.AggressiveInlining)] public static int Invoke(Vector256 result, Vector256 resultIndex) { - float curMin = float.MaxValue; - int curIn = 0; - for (int i = 0; i < Vector256.Count; i++) - { - if (result[i] == curMin && IsPositive(curMin) && !IsPositive(result[i])) - { - curMin = result[i]; - curIn = resultIndex[i]; - } - else if (result[i] < curMin) - { - curMin = result[i]; - curIn = resultIndex[i]; - } - } - return curIn; + // Min the upper/lower halves of the Vector256 + Vector128 resultLower = result.GetLower(); + Vector128 indexLower = resultIndex.GetLower(); + + Invoke(ref resultLower, result.GetUpper(), ref indexLower, resultIndex.GetUpper()); + return Invoke(resultLower, indexLower); } [MethodImpl(MethodImplOptions.AggressiveInlining)] public static void Invoke(ref Vector256 result, Vector256 current, ref Vector256 resultIndex, Vector256 curIndex) { - resultIndex = Vector256.ConditionalSelect(Vector256.Equals(result, current).AsInt32(), - Vector256.ConditionalSelect(IsPositive(result).AsInt32(), curIndex, resultIndex), - Vector256.ConditionalSelect(Vector256.LessThan(result, current).AsInt32(), resultIndex, curIndex)); + Vector256 lessThanMask = Vector256.LessThan(result, current); - result = Vector256.ConditionalSelect(Vector256.Equals(result, current), - Vector256.ConditionalSelect(IsPositive(result), current, result), - Vector256.Min(result, current)); - } + Vector256 equalMask = Vector256.Equals(result, current); - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static void Invoke(ref Vector256 result, Vector256 current, ref Vector256 resultIndex, Vector256 curIndex, Vector256 remainderMask) - { - resultIndex = Vector256.ConditionalSelect( - Vector256.Equals(remainderMask.AsInt32(), Vector256.Zero), - resultIndex, - Vector256.ConditionalSelect(Vector256.Equals(result, current).AsInt32(), - Vector256.ConditionalSelect(IsPositive(result).AsInt32(), curIndex, resultIndex), - Vector256.ConditionalSelect(Vector256.LessThan(result, current).AsInt32(), resultIndex, curIndex))); + if (equalMask.AsInt32() != Vector256.Zero) + { + Vector256 negativeMask = IsNegative(current); + Vector256 lessThanIndexMask = Vector256.LessThan(resultIndex, curIndex); + + lessThanMask |= (~negativeMask & equalMask) | (IsNegative(result) & equalMask & lessThanIndexMask.AsSingle()); + } + + lessThanMask = ~lessThanMask; - result = Vector256.ConditionalSelect( - Vector256.Equals(remainderMask, Vector256.Zero), - result, - MinOperator.Invoke(result, current)); + result = ElementWiseSelect(lessThanMask, result, current); + + resultIndex = ElementWiseSelect(lessThanMask.AsInt32(), resultIndex, curIndex); } #if NET8_0_OR_GREATER [MethodImpl(MethodImplOptions.AggressiveInlining)] public static int Invoke(Vector512 result, Vector512 resultIndex) { - float curMin = float.MaxValue; - int curIn = 0; - for (int i = 0; i < Vector512.Count; i++) - { - if (result[i] == curMin && IsPositive(curMin) && !IsPositive(result[i])) - { - curMin = result[i]; - curIn = resultIndex[i]; - } - else if (result[i] < curMin) - { - curMin = result[i]; - curIn = resultIndex[i]; - } - } - return curIn; + // Min the upper/lower halves of the Vector512 + Vector256 resultLower256 = result.GetLower(); + Vector256 indexLower256 = resultIndex.GetLower(); + + Invoke(ref resultLower256, result.GetUpper(), ref indexLower256, resultIndex.GetUpper()); + + // Min the upper/lower halves of the Vector256 + Vector128 resultLower128 = resultLower256.GetLower(); + Vector128 indexLower128 = indexLower256.GetLower(); + + Invoke(ref resultLower128, resultLower256.GetUpper(), ref indexLower128, indexLower256.GetUpper()); + + return Invoke(resultLower128, indexLower128); } [MethodImpl(MethodImplOptions.AggressiveInlining)] public static void Invoke(ref Vector512 result, Vector512 current, ref Vector512 resultIndex, Vector512 curIndex) { - resultIndex = Vector512.ConditionalSelect(Vector512.Equals(result, current).AsInt32(), - Vector512.ConditionalSelect(IsNegative(result).AsInt32(), curIndex, resultIndex), - Vector512.ConditionalSelect(Vector512.LessThan(result, current).AsInt32(), resultIndex, curIndex)); + Vector512 lessThanMask = Vector512.LessThan(result, current); - result = Vector512.ConditionalSelect(Vector512.Equals(result, current), - Vector512.ConditionalSelect(IsNegative(result), current, result), - Vector512.Min(result, current)); - } + Vector512 equalMask = Vector512.Equals(result, current); - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static void Invoke(ref Vector512 result, Vector512 current, ref Vector512 resultIndex, Vector512 curIndex, Vector512 remainderMask) - { - resultIndex = Vector512.ConditionalSelect( - Vector512.Equals(remainderMask.AsInt32(), Vector512.Zero), - resultIndex, - Vector512.ConditionalSelect(Vector512.Equals(result, current).AsInt32(), - Vector512.ConditionalSelect(IsPositive(result).AsInt32(), curIndex, resultIndex), - Vector512.ConditionalSelect(Vector512.LessThan(result, current).AsInt32(), resultIndex, curIndex))); + if (equalMask.AsInt32() != Vector512.Zero) + { + Vector512 negativeMask = IsNegative(current); + Vector512 lessThanIndexMask = Vector512.LessThan(resultIndex, curIndex); + + lessThanMask |= (~negativeMask & equalMask) | (IsNegative(result) & equalMask & lessThanIndexMask.AsSingle()); + } + + lessThanMask = ~lessThanMask; + + result = ElementWiseSelect(lessThanMask, result, current); - result = Vector512.ConditionalSelect( - Vector512.Equals(remainderMask, Vector512.Zero), - result, - MinOperator.Invoke(result, current)); + resultIndex = ElementWiseSelect(lessThanMask.AsInt32(), resultIndex, curIndex); } #endif } @@ -3996,22 +3870,16 @@ public static int Invoke(ReadOnlySpan result) [MethodImpl(MethodImplOptions.AggressiveInlining)] public static int Invoke(Vector128 result, Vector128 resultIndex) { - float curMin = result[0]; - int curIn = resultIndex[0]; - for (int i = 1; i < Vector128.Count; i++) - { - if (MathF.Abs(result[i]) == MathF.Abs(curMin) && IsPositive(curMin) && !IsPositive(result[i])) - { - curMin = result[i]; - curIn = resultIndex[i]; - } - else if (MathF.Abs(result[i]) < MathF.Abs(curMin)) - { - curMin = result[i]; - curIn = resultIndex[i]; - } - } - return curIn; + Vector128 tmp128 = Vector128.Shuffle(result, Vector128.Create(2, 3, 0, 1)); + Vector128 tmpIndex128 = Vector128.Shuffle(resultIndex, Vector128.Create(2, 3, 0, 1)); + + Invoke(ref result, tmp128, ref resultIndex, tmpIndex128); + + tmp128 = Vector128.Shuffle(result, Vector128.Create(1, 0, 3, 2)); + tmpIndex128 = Vector128.Shuffle(resultIndex, Vector128.Create(1, 0, 3, 2)); + + Invoke(ref result, tmp128, ref resultIndex, tmpIndex128); + return resultIndex.ToScalar(); } [MethodImpl(MethodImplOptions.AggressiveInlining)] @@ -4019,52 +3887,34 @@ public static void Invoke(ref Vector128 result, Vector128 current, { Vector128 minMag = Vector128.Abs(result), currentMag = Vector128.Abs(current); - resultIndex = Vector128.ConditionalSelect(Vector128.Equals(result, current).AsInt32(), - Vector128.ConditionalSelect(IsPositive(result).AsInt32(), curIndex, resultIndex), - Vector128.ConditionalSelect(Vector128.LessThan(minMag, currentMag).AsInt32(), resultIndex, curIndex)); + Vector128 lessThanMask = Vector128.LessThan(minMag, currentMag); - result = Vector128.ConditionalSelect(Vector128.Equals(result, current), - Vector128.ConditionalSelect(IsPositive(result), current, result), - Vector128.Min(minMag, currentMag)); - } + Vector128 equalMask = Vector128.Equals(result, current); - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static void Invoke(ref Vector128 result, Vector128 current, ref Vector128 resultIndex, Vector128 curIndex, Vector128 remainderMask) - { - Vector128 minMag = Vector128.Abs(result), currentMag = Vector128.Abs(current); + if (equalMask.AsInt32() != Vector128.Zero) + { + Vector128 negativeMask = IsNegative(current); + Vector128 lessThanIndexMask = Vector128.LessThan(resultIndex, curIndex); - resultIndex = Vector128.ConditionalSelect( - Vector128.Equals(remainderMask.AsInt32(), Vector128.Zero), - resultIndex, - Vector128.ConditionalSelect(Vector128.Equals(result, current).AsInt32(), - Vector128.ConditionalSelect(IsPositive(result).AsInt32(), curIndex, resultIndex), - Vector128.ConditionalSelect(Vector128.LessThan(minMag, currentMag).AsInt32(), resultIndex, curIndex))); + lessThanMask |= (~negativeMask & equalMask) | (IsNegative(result) & equalMask & lessThanIndexMask.AsSingle()); + } + + lessThanMask = ~lessThanMask; - result = Vector128.ConditionalSelect( - Vector128.Equals(remainderMask, Vector128.Zero), - result, - MinOperator.Invoke(minMag, currentMag)); + result = ElementWiseSelect(lessThanMask, result, current); + + resultIndex = ElementWiseSelect(lessThanMask.AsInt32(), resultIndex, curIndex); } [MethodImpl(MethodImplOptions.AggressiveInlining)] public static int Invoke(Vector256 result, Vector256 resultIndex) { - float curMin = result[0]; - int curIn = resultIndex[0]; - for (int i = 1; i < Vector256.Count; i++) - { - if (MathF.Abs(result[i]) == MathF.Abs(curMin) && IsPositive(curMin) && !IsPositive(result[i])) - { - curMin = result[i]; - curIn = resultIndex[i]; - } - else if (MathF.Abs(result[i]) < MathF.Abs(curMin)) - { - curMin = result[i]; - curIn = resultIndex[i]; - } - } - return curIn; + // Min the upper/lower halves of the Vector256 + Vector128 resultLower = result.GetLower(); + Vector128 indexLower = resultIndex.GetLower(); + + Invoke(ref resultLower, result.GetUpper(), ref indexLower, resultIndex.GetUpper()); + return Invoke(resultLower, indexLower); } [MethodImpl(MethodImplOptions.AggressiveInlining)] @@ -4072,53 +3922,42 @@ public static void Invoke(ref Vector256 result, Vector256 current, { Vector256 minMag = Vector256.Abs(result), currentMag = Vector256.Abs(current); - resultIndex = Vector256.ConditionalSelect(Vector256.Equals(result, current).AsInt32(), - Vector256.ConditionalSelect(IsPositive(result).AsInt32(), curIndex, resultIndex), - Vector256.ConditionalSelect(Vector256.LessThan(minMag, currentMag).AsInt32(), resultIndex, curIndex)); + Vector256 lessThanMask = Vector256.LessThan(minMag, currentMag); - result = Vector256.ConditionalSelect(Vector256.Equals(result, current), - Vector256.ConditionalSelect(IsPositive(result), current, result), - Vector256.Min(minMag, currentMag)); - } + Vector256 equalMask = Vector256.Equals(result, current); - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static void Invoke(ref Vector256 result, Vector256 current, ref Vector256 resultIndex, Vector256 curIndex, Vector256 remainderMask) - { - Vector256 minMag = Vector256.Abs(result), currentMag = Vector256.Abs(current); + if (equalMask.AsInt32() != Vector256.Zero) + { + Vector256 negativeMask = IsNegative(current); + Vector256 lessThanIndexMask = Vector256.LessThan(resultIndex, curIndex); - resultIndex = Vector256.ConditionalSelect( - Vector256.Equals(remainderMask.AsInt32(), Vector256.Zero), - resultIndex, - Vector256.ConditionalSelect(Vector256.Equals(result, current).AsInt32(), - Vector256.ConditionalSelect(IsPositive(result).AsInt32(), curIndex, resultIndex), - Vector256.ConditionalSelect(Vector256.LessThan(minMag, currentMag).AsInt32(), resultIndex, curIndex))); + lessThanMask |= (~negativeMask & equalMask) | (IsNegative(result) & equalMask & lessThanIndexMask.AsSingle()); + } + + lessThanMask = ~lessThanMask; - result = Vector256.ConditionalSelect( - Vector256.Equals(remainderMask, Vector256.Zero), - result, - MinOperator.Invoke(minMag, currentMag)); + result = ElementWiseSelect(lessThanMask, result, current); + + resultIndex = ElementWiseSelect(lessThanMask.AsInt32(), resultIndex, curIndex); } #if NET8_0_OR_GREATER [MethodImpl(MethodImplOptions.AggressiveInlining)] public static int Invoke(Vector512 result, Vector512 resultIndex) { - float curMin = result[0]; - int curIn = resultIndex[0]; - for (int i = 1; i < Vector512.Count; i++) - { - if (MathF.Abs(result[i]) == MathF.Abs(curMin) && IsPositive(curMin) && !IsPositive(result[i])) - { - curMin = result[i]; - curIn = resultIndex[i]; - } - else if (MathF.Abs(result[i]) < MathF.Abs(curMin)) - { - curMin = result[i]; - curIn = resultIndex[i]; - } - } - return curIn; + // Min the upper/lower halves of the Vector512 + Vector256 resultLower256 = result.GetLower(); + Vector256 indexLower256 = resultIndex.GetLower(); + + Invoke(ref resultLower256, result.GetUpper(), ref indexLower256, resultIndex.GetUpper()); + + // Min the upper/lower halves of the Vector256 + Vector128 resultLower128 = resultLower256.GetLower(); + Vector128 indexLower128 = indexLower256.GetLower(); + + Invoke(ref resultLower128, resultLower256.GetUpper(), ref indexLower128, indexLower256.GetUpper()); + + return Invoke(resultLower128, indexLower128); } [MethodImpl(MethodImplOptions.AggressiveInlining)] @@ -4126,31 +3965,23 @@ public static void Invoke(ref Vector512 result, Vector512 current, { Vector512 minMag = Vector512.Abs(result), currentMag = Vector512.Abs(current); - resultIndex = Vector512.ConditionalSelect(Vector512.Equals(result, current).AsInt32(), - Vector512.ConditionalSelect(IsNegative(result).AsInt32(), curIndex, resultIndex), - Vector512.ConditionalSelect(Vector512.LessThan(minMag, currentMag).AsInt32(), resultIndex, curIndex)); + Vector512 lessThanMask = Vector512.LessThan(minMag, currentMag); - result = Vector512.ConditionalSelect(Vector512.Equals(result, current), - Vector512.ConditionalSelect(IsNegative(result), current, result), - Vector512.Min(minMag, currentMag)); - } + Vector512 equalMask = Vector512.Equals(result, current); - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static void Invoke(ref Vector512 result, Vector512 current, ref Vector512 resultIndex, Vector512 curIndex, Vector512 remainderMask) - { - Vector512 minMag = Vector512.Abs(result), currentMag = Vector512.Abs(current); + if (equalMask.AsInt32() != Vector512.Zero) + { + Vector512 negativeMask = IsNegative(current); + Vector512 lessThanIndexMask = Vector512.LessThan(resultIndex, curIndex); - resultIndex = Vector512.ConditionalSelect( - Vector512.Equals(remainderMask.AsInt32(), Vector512.Zero), - resultIndex, - Vector512.ConditionalSelect(Vector512.Equals(result, current).AsInt32(), - Vector512.ConditionalSelect(IsPositive(result).AsInt32(), curIndex, resultIndex), - Vector512.ConditionalSelect(Vector512.LessThan(minMag, currentMag).AsInt32(), resultIndex, curIndex))); + lessThanMask |= (~negativeMask & equalMask) | (IsNegative(result) & equalMask & lessThanIndexMask.AsSingle()); + } + + lessThanMask = ~lessThanMask; - result = Vector512.ConditionalSelect( - Vector512.Equals(remainderMask, Vector512.Zero), - result, - MinOperator.Invoke(minMag, currentMag)); + result = ElementWiseSelect(lessThanMask, result, current); + + resultIndex = ElementWiseSelect(lessThanMask.AsInt32(), resultIndex, curIndex); } #endif } @@ -5562,6 +5393,62 @@ public static Vector512 Invoke(Vector512 x) #endif } + private static Vector128 ElementWiseSelect(Vector128 mask, Vector128 left, Vector128 right) + { + if (Sse41.IsSupported) + { + return Sse41.BlendVariable(left, right, mask); + } + else return Vector128.ConditionalSelect(mask, left, right); + } + + private static Vector128 ElementWiseSelect(Vector128 mask, Vector128 left, Vector128 right) + { + if (Sse41.IsSupported) + { + return Sse41.BlendVariable(left, right, mask); + } + else return Vector128.ConditionalSelect(mask, left, right); + } + + private static Vector256 ElementWiseSelect(Vector256 mask, Vector256 left, Vector256 right) + { + if (Avx2.IsSupported) + { + return Avx2.BlendVariable(left, right, mask); + } + else return Vector256.ConditionalSelect(mask, left, right); + } + + private static Vector256 ElementWiseSelect(Vector256 mask, Vector256 left, Vector256 right) + { + if (Avx2.IsSupported) + { + return Avx2.BlendVariable(left, right, mask); + } + else return Vector256.ConditionalSelect(mask, left, right); + } + +#if NET8_0_OR_GREATER + private static Vector512 ElementWiseSelect(Vector512 mask, Vector512 left, Vector512 right) + { + if (Avx512F.IsSupported) + { + return Avx512F.BlendVariable(left, right, mask); + } + else return Vector512.ConditionalSelect(mask, left, right); + } + + private static Vector512 ElementWiseSelect(Vector512 mask, Vector512 left, Vector512 right) + { + if (Avx512F.IsSupported) + { + return Avx512F.BlendVariable(left, right, mask); + } + else return Vector512.ConditionalSelect(mask, left, right); + } +#endif + /// 1f / (1f + MathF.Exp(-x)) private readonly struct SigmoidOperator : IUnaryOperator { From e143ff7bb900382e317868bfc8ab2927a395f810 Mon Sep 17 00:00:00 2001 From: Michael Sharp Date: Wed, 18 Oct 2023 14:39:41 -0600 Subject: [PATCH 10/19] net standard updated --- .../Tensors/TensorPrimitives.netcore.cs | 36 +--- .../Tensors/TensorPrimitives.netstandard.cs | 186 +++++++----------- 2 files changed, 73 insertions(+), 149 deletions(-) diff --git a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/TensorPrimitives.netcore.cs b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/TensorPrimitives.netcore.cs index 753d8d73cbed22..23ad1edc543c83 100644 --- a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/TensorPrimitives.netcore.cs +++ b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/TensorPrimitives.netcore.cs @@ -8163,8 +8163,6 @@ public static void Invoke(ref Vector128 max, Vector128 current, re greaterThanMask |= (negativeMask & equalMask) | (~IsNegative(max) & equalMask & lessThanMask.AsSingle()); } - greaterThanMask = ~greaterThanMask; - max = ElementWiseSelect(greaterThanMask, max, current); maxIndex = ElementWiseSelect(greaterThanMask.AsInt32(), maxIndex, curIndex); @@ -8196,8 +8194,6 @@ public static void Invoke(ref Vector256 max, Vector256 current, re greaterThanMask |= (negativeMask & equalMask) | (~IsNegative(max) & equalMask & lessThanMask.AsSingle()); } - greaterThanMask = ~greaterThanMask; - max = ElementWiseSelect(greaterThanMask, max, current); maxIndex = ElementWiseSelect(greaterThanMask.AsInt32(), maxIndex, curIndex); @@ -8237,8 +8233,6 @@ public static void Invoke(ref Vector512 max, Vector512 current, re greaterThanMask |= (negativeMask & equalMask) | (~IsNegative(max) & equalMask & lessThanMask.AsSingle()); } - greaterThanMask = ~greaterThanMask; - max = ElementWiseSelect(greaterThanMask, max, current); maxIndex = ElementWiseSelect(greaterThanMask.AsInt32(), maxIndex, curIndex); @@ -8316,8 +8310,6 @@ public static void Invoke(ref Vector128 max, Vector128 current, re greaterThanMask |= (negativeMask & equalMask) | (~IsNegative(max) & equalMask & lessThanMask.AsSingle()); } - greaterThanMask = ~greaterThanMask; - max = ElementWiseSelect(greaterThanMask, max, current); maxIndex = ElementWiseSelect(greaterThanMask.AsInt32(), maxIndex, curIndex); @@ -8351,8 +8343,6 @@ public static void Invoke(ref Vector256 max, Vector256 current, re greaterThanMask |= (negativeMask & equalMask) | (~IsNegative(max) & equalMask & lessThanMask.AsSingle()); } - greaterThanMask = ~greaterThanMask; - max = ElementWiseSelect(greaterThanMask, max, current); maxIndex = ElementWiseSelect(greaterThanMask.AsInt32(), maxIndex, curIndex); @@ -8393,8 +8383,6 @@ public static void Invoke(ref Vector512 max, Vector512 current, re greaterThanMask |= (negativeMask & equalMask) | (~IsNegative(max) & equalMask & lessThanMask.AsSingle()); } - greaterThanMask = ~greaterThanMask; - max = ElementWiseSelect(greaterThanMask, max, current); maxIndex = ElementWiseSelect(greaterThanMask.AsInt32(), maxIndex, curIndex); @@ -8471,8 +8459,6 @@ public static void Invoke(ref Vector128 result, Vector128 current, lessThanMask |= (~negativeMask & equalMask) | (IsNegative(result) & equalMask & lessThanIndexMask.AsSingle()); } - lessThanMask = ~lessThanMask; - result = ElementWiseSelect(lessThanMask, result, current); resultIndex = ElementWiseSelect(lessThanMask.AsInt32(), resultIndex, curIndex); @@ -8505,8 +8491,6 @@ public static void Invoke(ref Vector256 result, Vector256 current, lessThanMask |= (~negativeMask & equalMask) | (IsNegative(result) & equalMask & lessThanIndexMask.AsSingle()); } - lessThanMask = ~lessThanMask; - result = ElementWiseSelect(lessThanMask, result, current); resultIndex = ElementWiseSelect(lessThanMask.AsInt32(), resultIndex, curIndex); @@ -8546,8 +8530,6 @@ public static void Invoke(ref Vector512 result, Vector512 current, lessThanMask |= (~negativeMask & equalMask) | (IsNegative(result) & equalMask & lessThanIndexMask.AsSingle()); } - lessThanMask = ~lessThanMask; - result = ElementWiseSelect(lessThanMask, result, current); resultIndex = ElementWiseSelect(lessThanMask.AsInt32(), resultIndex, curIndex); @@ -8625,8 +8607,6 @@ public static void Invoke(ref Vector128 result, Vector128 current, lessThanMask |= (~negativeMask & equalMask) | (IsNegative(result) & equalMask & lessThanIndexMask.AsSingle()); } - lessThanMask = ~lessThanMask; - result = ElementWiseSelect(lessThanMask, result, current); resultIndex = ElementWiseSelect(lessThanMask.AsInt32(), resultIndex, curIndex); @@ -8660,8 +8640,6 @@ public static void Invoke(ref Vector256 result, Vector256 current, lessThanMask |= (~negativeMask & equalMask) | (IsNegative(result) & equalMask & lessThanIndexMask.AsSingle()); } - lessThanMask = ~lessThanMask; - result = ElementWiseSelect(lessThanMask, result, current); resultIndex = ElementWiseSelect(lessThanMask.AsInt32(), resultIndex, curIndex); @@ -8703,8 +8681,6 @@ public static void Invoke(ref Vector512 result, Vector512 current, lessThanMask |= (~negativeMask & equalMask) | (IsNegative(result) & equalMask & lessThanIndexMask.AsSingle()); } - lessThanMask = ~lessThanMask; - result = ElementWiseSelect(lessThanMask, result, current); resultIndex = ElementWiseSelect(lessThanMask.AsInt32(), resultIndex, curIndex); @@ -10123,7 +10099,7 @@ private static Vector128 ElementWiseSelect(Vector128 mask, Vector1 { if (Sse41.IsSupported) { - return Sse41.BlendVariable(left, right, mask); + return Sse41.BlendVariable(left, right, ~mask); } else return Vector128.ConditionalSelect(mask, left, right); } @@ -10132,7 +10108,7 @@ private static Vector128 ElementWiseSelect(Vector128 mask, Vector128 ElementWiseSelect(Vector256 mask, Vector2 { if (Avx2.IsSupported) { - return Avx2.BlendVariable(left, right, mask); + return Avx2.BlendVariable(left, right, ~mask); } else return Vector256.ConditionalSelect(mask, left, right); } @@ -10150,7 +10126,7 @@ private static Vector256 ElementWiseSelect(Vector256 mask, Vector256 ElementWiseSelect(Vector512 mask, Vector5 { if (Avx512F.IsSupported) { - return Avx512F.BlendVariable(left, right, mask); + return Avx512F.BlendVariable(left, right, ~mask); } else return Vector512.ConditionalSelect(mask, left, right); } @@ -10169,7 +10145,7 @@ private static Vector512 ElementWiseSelect(Vector512 mask, Vector512(ReadOnlySpan incrementArray[j] = Vector.Count; } - - Span resultIndexSpan = new Span(resultIndexArray); - Span curIndexSpan = new Span(curIndexArray); - Span incrementSpan = new Span(incrementArray); - - - Vector resultIndex = AsVector(ref MemoryMarshal.GetReference(resultIndexSpan), 0); - Vector curIndex = AsVector(ref MemoryMarshal.GetReference(curIndexSpan), 0); - Vector increment = AsVector(ref MemoryMarshal.GetReference(incrementSpan), 0); + Vector resultIndex = new Vector(resultIndexArray); + Vector curIndex = new Vector(curIndexArray); + Vector increment = new Vector(incrementArray); // Load the first vector as the initial set of results, and bail immediately // to scalar handling if it contains any NaNs (which don't compare equally to themselves). @@ -379,18 +373,17 @@ private static int IndexOfMinMaxCore(ReadOnlySpan { for(int j = 0; j < Vector.Count; j++) { - incrementSpan[j] = x.Length - i; + incrementArray[j] = x.Length - i; } - curIndex = Vector.Add(curIndex, AsVector(ref MemoryMarshal.GetReference(incrementSpan), 0)); + curIndex = Vector.Add(curIndex, AsVector(ref MemoryMarshal.GetReference(incrementArray.AsSpan()), 0)); current = AsVector(ref xRef, x.Length - Vector.Count); if (!Vector.EqualsAll(current, current)) { goto Scalar; } - Vector remainderMask = CreateRemainderMaskSingleVector(x.Length - i); - op.Invoke(ref resultVector, current, ref resultIndex, curIndex, remainderMask); + op.Invoke(ref resultVector, current, ref resultIndex, curIndex); } result = op.Invoke(resultVector, resultIndex); @@ -2462,7 +2455,6 @@ private interface IIndexOfOperator int Invoke(ReadOnlySpan result); int Invoke(Vector result, Vector resultIndex); void Invoke(ref Vector result, Vector current, ref Vector resultIndex, Vector curIndex); - void Invoke(ref Vector result, Vector current, ref Vector resultIndex, Vector curIndex, Vector remainderMask); } /// MathF.Max(x, y) (but without guaranteed NaN propagation) @@ -2505,54 +2497,44 @@ public int Invoke(ReadOnlySpan result) } [MethodImpl(MethodImplOptions.AggressiveInlining)] - public int Invoke(Vector result, Vector maxIndex) + public int Invoke(Vector result, Vector resultIndex) { - float curMax = float.MinValue; - int curIn = 0; - for (int i = 0; i < Vector.Count; i++) + float curMax = result[0]; + int curIn = resultIndex[0]; + for (int i = 1; i < Vector.Count; i++) { if (result[i] == curMax && IsNegative(curMax) && !IsNegative(result[i])) { curMax = result[i]; - curIn = maxIndex[i]; + curIn = resultIndex[i]; } else if (result[i] > curMax) { curMax = result[i]; - curIn = maxIndex[i]; + curIn = resultIndex[i]; } } return curIn; } [MethodImpl(MethodImplOptions.AggressiveInlining)] - public void Invoke(ref Vector max, Vector current, ref Vector maxIndex, Vector curIndex) + public void Invoke(ref Vector result, Vector current, ref Vector resultIndex, Vector curIndex) { - maxIndex = Vector.ConditionalSelect(Vector.Equals(max, current), - Vector.ConditionalSelect((Vector)IsNegative(max), curIndex, maxIndex), - Vector.ConditionalSelect(Vector.GreaterThan(max, current), maxIndex, curIndex)); + Vector lessThanMask = Vector.GreaterThan(result, current); - max = Vector.ConditionalSelect(Vector.Equals(max, current), - Vector.ConditionalSelect(IsNegative(max), current, max), - Vector.Max(max, current)); - } + Vector equalMask = Vector.Equals(result, current); - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public void Invoke(ref Vector max, Vector current, ref Vector maxIndex, Vector curIndex, Vector remainderMask) - { - maxIndex = Vector.ConditionalSelect( - Vector.Equals((Vector)remainderMask, Vector.Zero), - maxIndex, - Vector.ConditionalSelect(Vector.Equals(max, current), - Vector.ConditionalSelect((Vector)IsNegative(max), curIndex, maxIndex), - Vector.ConditionalSelect(Vector.GreaterThan(max, current), maxIndex, curIndex))); - - max = Vector.ConditionalSelect( - Vector.Equals(remainderMask, Vector.Zero), - max, - Vector.ConditionalSelect(Vector.Equals(max, current), - Vector.ConditionalSelect(IsNegative(max), current, max), - Vector.Max(max, current))); + if (equalMask != Vector.Zero) + { + Vector negativeMask = IsNegative(current); + Vector lessThanIndexMask = Vector.LessThan(resultIndex, curIndex); + + lessThanMask |= ((Vector)~negativeMask & equalMask) | ((Vector)IsNegative(result) & equalMask & lessThanIndexMask); + } + + result = Vector.ConditionalSelect(lessThanMask, result, current); + + resultIndex = Vector.ConditionalSelect(lessThanMask, resultIndex, curIndex); } } @@ -2616,37 +2598,25 @@ public int Invoke(Vector result, Vector maxIndex) } [MethodImpl(MethodImplOptions.AggressiveInlining)] - public void Invoke(ref Vector max, Vector current, ref Vector maxIndex, Vector curIndex) + public void Invoke(ref Vector result, Vector current, ref Vector resultIndex, Vector curIndex) { - Vector maxMag = Vector.Abs(max), currentMag = Vector.Abs(current); + Vector maxMag = Vector.Abs(result), currentMag = Vector.Abs(current); - maxIndex = Vector.ConditionalSelect(Vector.Equals(max, current), - Vector.ConditionalSelect((Vector)IsNegative(max), curIndex, maxIndex), - Vector.ConditionalSelect(Vector.GreaterThan(maxMag, currentMag), maxIndex, curIndex)); + Vector lessThanMask = Vector.GreaterThan(maxMag, currentMag); - max = Vector.ConditionalSelect(Vector.Equals(max, current), - Vector.ConditionalSelect(IsNegative(max), current, max), - Vector.Max(maxMag, currentMag)); - } + Vector equalMask = Vector.Equals(result, current); - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public void Invoke(ref Vector max, Vector current, ref Vector maxIndex, Vector curIndex, Vector remainderMask) - { - Vector maxMag = Vector.Abs(max), currentMag = Vector.Abs(current); + if (equalMask != Vector.Zero) + { + Vector negativeMask = IsNegative(current); + Vector lessThanIndexMask = Vector.LessThan(resultIndex, curIndex); - maxIndex = Vector.ConditionalSelect( - Vector.Equals(((Vector)remainderMask), Vector.Zero), - maxIndex, - Vector.ConditionalSelect(Vector.Equals(max, current), - Vector.ConditionalSelect((Vector)IsNegative(max), curIndex, maxIndex), - Vector.ConditionalSelect(Vector.GreaterThan(maxMag, currentMag), maxIndex, curIndex))); + lessThanMask |= ((Vector)~negativeMask & equalMask) | ((Vector)IsNegative(result) & equalMask & lessThanIndexMask); + } + + result = Vector.ConditionalSelect(lessThanMask, result, current); - max = Vector.ConditionalSelect( - Vector.Equals(remainderMask, Vector.Zero), - max, - Vector.ConditionalSelect(Vector.Equals(max, current), - Vector.ConditionalSelect(IsNegative(max), current, max), - Vector.Max(maxMag, currentMag))); + resultIndex = Vector.ConditionalSelect(lessThanMask, resultIndex, curIndex); } } @@ -2691,9 +2661,9 @@ public int Invoke(ReadOnlySpan result) [MethodImpl(MethodImplOptions.AggressiveInlining)] public int Invoke(Vector result, Vector resultIndex) { - float curMin = float.MaxValue; - int curIn = 0; - for (int i = 0; i < Vector.Count; i++) + float curMin = result[0]; + int curIn = resultIndex[0]; + for (int i = 1; i < Vector.Count; i++) { if (result[i] == curMin && IsPositive(curMin) && !IsPositive(result[i])) { @@ -2712,31 +2682,21 @@ public int Invoke(Vector result, Vector resultIndex) [MethodImpl(MethodImplOptions.AggressiveInlining)] public void Invoke(ref Vector result, Vector current, ref Vector resultIndex, Vector curIndex) { - resultIndex = Vector.ConditionalSelect(Vector.Equals(result, current), - Vector.ConditionalSelect(((Vector)IsPositive(result)), curIndex, resultIndex), - Vector.ConditionalSelect(Vector.LessThan(result, current), resultIndex, curIndex)); + Vector lessThanMask = Vector.LessThan(result, current); - result = Vector.ConditionalSelect(Vector.Equals(result, current), - Vector.ConditionalSelect(IsPositive(result), current, result), - Vector.Min(result, current)); - } + Vector equalMask = Vector.Equals(result, current); - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public void Invoke(ref Vector result, Vector current, ref Vector resultIndex, Vector curIndex, Vector remainderMask) - { - resultIndex = Vector.ConditionalSelect( - Vector.Equals(((Vector)remainderMask), Vector.Zero), - resultIndex, - Vector.ConditionalSelect(Vector.Equals(result, current), - Vector.ConditionalSelect((Vector)IsPositive(result), curIndex, resultIndex), - Vector.ConditionalSelect(Vector.LessThan(result, current), resultIndex, curIndex))); - - result = Vector.ConditionalSelect( - Vector.Equals(remainderMask, Vector.Zero), - result, - Vector.ConditionalSelect(Vector.Equals(result, current), - Vector.ConditionalSelect(IsPositive(result), current, result), - Vector.Min(result, current))); + if (equalMask != Vector.Zero) + { + Vector negativeMask = IsNegative(current); + Vector lessThanIndexMask = Vector.LessThan(resultIndex, curIndex); + + lessThanMask |= ((Vector)negativeMask & equalMask) | (~(Vector)IsNegative(result) & equalMask & lessThanIndexMask); + } + + result = Vector.ConditionalSelect(lessThanMask, result, current); + + resultIndex = Vector.ConditionalSelect(lessThanMask, resultIndex, curIndex); } } @@ -2804,33 +2764,21 @@ public void Invoke(ref Vector result, Vector current, ref Vector minMag = Vector.Abs(result), currentMag = Vector.Abs(current); - resultIndex = Vector.ConditionalSelect(Vector.Equals(result, current), - Vector.ConditionalSelect(((Vector)IsPositive(result)), curIndex, resultIndex), - Vector.ConditionalSelect(Vector.LessThan(minMag, currentMag), resultIndex, curIndex)); + Vector lessThanMask = Vector.LessThan(minMag, currentMag); - result = Vector.ConditionalSelect(Vector.Equals(result, current), - Vector.ConditionalSelect(IsPositive(result), current, result), - Vector.Min(minMag, currentMag)); - } + Vector equalMask = Vector.Equals(result, current); - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public void Invoke(ref Vector result, Vector current, ref Vector resultIndex, Vector curIndex, Vector remainderMask) - { - Vector minMag = Vector.Abs(result), currentMag = Vector.Abs(current); + if (equalMask != Vector.Zero) + { + Vector negativeMask = IsNegative(current); + Vector lessThanIndexMask = Vector.LessThan(resultIndex, curIndex); + + lessThanMask |= ((Vector)negativeMask & equalMask) | (~(Vector)IsNegative(result) & equalMask & lessThanIndexMask); + } + + result = Vector.ConditionalSelect(lessThanMask, result, current); - resultIndex = Vector.ConditionalSelect( - Vector.Equals(((Vector)remainderMask), Vector.Zero), - resultIndex, - Vector.ConditionalSelect(Vector.Equals(result, current), - Vector.ConditionalSelect(((Vector)IsPositive(result)), curIndex, resultIndex), - Vector.ConditionalSelect(Vector.LessThan(minMag, currentMag), resultIndex, curIndex))); - - result = Vector.ConditionalSelect( - Vector.Equals(remainderMask, Vector.Zero), - result, - Vector.ConditionalSelect(Vector.Equals(result, current), - Vector.ConditionalSelect(IsPositive(result), current, result), - Vector.Min(minMag, currentMag))); + resultIndex = Vector.ConditionalSelect(lessThanMask, resultIndex, curIndex); } } From 7a91a73d45717c1b75d6a4b5b92af3cab7257382 Mon Sep 17 00:00:00 2001 From: Michael Sharp Date: Wed, 18 Oct 2023 14:41:58 -0600 Subject: [PATCH 11/19] add reference assembly exclusions --- .../System.Numerics.Tensors/src/System.Numerics.Tensors.csproj | 1 + 1 file changed, 1 insertion(+) diff --git a/src/libraries/System.Numerics.Tensors/src/System.Numerics.Tensors.csproj b/src/libraries/System.Numerics.Tensors/src/System.Numerics.Tensors.csproj index b0449170860b45..b87938ff24586b 100644 --- a/src/libraries/System.Numerics.Tensors/src/System.Numerics.Tensors.csproj +++ b/src/libraries/System.Numerics.Tensors/src/System.Numerics.Tensors.csproj @@ -10,6 +10,7 @@ Once this package has shipped a stable version, the following line should be removed in order to re-enable validation. --> true + ReferenceAssemblyExclusions.txt From b0a06858ef4a89e4b33a8049d670a2527f3312ae Mon Sep 17 00:00:00 2001 From: Michael Sharp Date: Wed, 18 Oct 2023 21:45:58 -0600 Subject: [PATCH 12/19] made naive approach better --- .../Tensors/TensorPrimitives.netcore.cs | 16 ++++++++++++---- .../Tensors/TensorPrimitives.netstandard.cs | 18 +++++++++++++----- 2 files changed, 25 insertions(+), 9 deletions(-) diff --git a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/TensorPrimitives.netcore.cs b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/TensorPrimitives.netcore.cs index 23ad1edc543c83..7fcf047650b2ed 100644 --- a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/TensorPrimitives.netcore.cs +++ b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/TensorPrimitives.netcore.cs @@ -8246,6 +8246,7 @@ public static void Invoke(ref Vector512 max, Vector512 current, re public static int Invoke(ReadOnlySpan result) { float curMax = result[0]; + float curMaxAbs = MathF.Abs(curMax); int curIn = 0; if (float.IsNaN(curMax)) { @@ -8255,22 +8256,25 @@ public static int Invoke(ReadOnlySpan result) for (int i = 1; i < result.Length; i++) { float current = result[i]; + float currentAbs = MathF.Abs(current); if (float.IsNaN(current)) { return i; } - if (MathF.Abs(curMax) == MathF.Abs(current)) + if (curMaxAbs == currentAbs) { if (IsNegative(curMax) && !IsNegative(current)) { curMax = current; + curMaxAbs = MathF.Abs(current); curIn = i; } } - else if (MathF.Abs(current) > MathF.Abs(curMax)) + else if (currentAbs > curMaxAbs) { curMax = current; + curMaxAbs = MathF.Abs(current); curIn = i; } } @@ -8543,6 +8547,7 @@ public static void Invoke(ref Vector512 result, Vector512 current, public static int Invoke(ReadOnlySpan result) { float curMin = result[0]; + float curMinAbs = MathF.Abs(curMin); int curIn = 0; if (float.IsNaN(curMin)) { @@ -8552,22 +8557,25 @@ public static int Invoke(ReadOnlySpan result) for (int i = 1; i < result.Length; i++) { float current = result[i]; + float currentAbs = MathF.Abs(current); if (float.IsNaN(current)) { return i; } - if (MathF.Abs(curMin) == MathF.Abs(current)) + if (curMinAbs == currentAbs) { if (IsPositive(curMin) && !IsPositive(current)) { curMin = current; + curMinAbs = MathF.Abs(current); curIn = i; } } - else if (MathF.Abs(current) < MathF.Abs(curMin)) + else if (currentAbs < curMinAbs) { curMin = current; + curMinAbs = MathF.Abs(current); curIn = i; } } diff --git a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/TensorPrimitives.netstandard.cs b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/TensorPrimitives.netstandard.cs index 12e12620541846..764f33c358c853 100644 --- a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/TensorPrimitives.netstandard.cs +++ b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/TensorPrimitives.netstandard.cs @@ -2544,6 +2544,7 @@ public void Invoke(ref Vector result, Vector current, ref Vector result) { float curMax = result[0]; + float curMaxAbs = MathF.Abs(curMax); int curIn = 0; if (float.IsNaN(curMax)) { @@ -2553,22 +2554,25 @@ public int Invoke(ReadOnlySpan result) for (int i = 1; i < result.Length; i++) { float current = result[i]; + float currentAbs = MathF.Abs(current); if (float.IsNaN(current)) { return i; } - if (MathF.Abs(curMax) == MathF.Abs(current)) + if (curMaxAbs == currentAbs) { if (IsNegative(curMax) && !IsNegative(current)) { curMax = current; + curMaxAbs = MathF.Abs(current); curIn = i; } } - else if (MathF.Abs(current) > MathF.Abs(curMax)) + else if (currentAbs > curMaxAbs) { curMax = current; + curMaxAbs = MathF.Abs(current); curIn = i; } } @@ -2705,7 +2709,8 @@ public void Invoke(ref Vector result, Vector current, ref Vector result) { - float curMin = result[0]; + float curMin = result[0]; + float curMinAbs = MathF.Abs(curMin); int curIn = 0; if (float.IsNaN(curMin)) { @@ -2715,22 +2720,25 @@ public int Invoke(ReadOnlySpan result) for (int i = 1; i < result.Length; i++) { float current = result[i]; + float currentAbs = MathF.Abs(current); if (float.IsNaN(current)) { return i; } - if (MathF.Abs(curMin) == MathF.Abs(current)) + if (curMinAbs == currentAbs) { if (IsPositive(curMin) && !IsPositive(current)) { curMin = current; + curMinAbs = MathF.Abs(current); curIn = i; } } - else if (MathF.Abs(current) < MathF.Abs(curMin)) + else if (currentAbs < curMinAbs) { curMin = current; + curMinAbs = MathF.Abs(current); curIn = i; } } From c6818557ae767ddcc8502604f1beb4c7e2a915b6 Mon Sep 17 00:00:00 2001 From: Michael Sharp Date: Thu, 19 Oct 2023 09:29:20 -0600 Subject: [PATCH 13/19] resolved PR comments --- .../Numerics/Tensors/TensorPrimitives.cs | 4 +++ .../Tensors/TensorPrimitives.netstandard.cs | 30 ++++--------------- 2 files changed, 10 insertions(+), 24 deletions(-) diff --git a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/TensorPrimitives.cs b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/TensorPrimitives.cs index 766c242f1c246a..69562bee25124d 100644 --- a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/TensorPrimitives.cs +++ b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/TensorPrimitives.cs @@ -320,6 +320,7 @@ public static int IndexOfMax(ReadOnlySpan x) { return -1; } + return IndexOfMinMaxCore(x); } @@ -343,6 +344,7 @@ public static int IndexOfMaxMagnitude(ReadOnlySpan x) { return -1; } + return IndexOfMinMaxCore(x); } @@ -365,6 +367,7 @@ public static int IndexOfMin(ReadOnlySpan x) { return -1; } + return IndexOfMinMaxCore(x); } @@ -388,6 +391,7 @@ public static int IndexOfMinMagnitude(ReadOnlySpan x) { return -1; } + return IndexOfMinMaxCore(x); } diff --git a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/TensorPrimitives.netstandard.cs b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/TensorPrimitives.netstandard.cs index 764f33c358c853..e26379c2e72b3d 100644 --- a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/TensorPrimitives.netstandard.cs +++ b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/TensorPrimitives.netstandard.cs @@ -311,38 +311,24 @@ private static float MinMaxCore(ReadOnlySpan x, TMinMaxO return result; } - /// - /// This is the same as - /// with an identity transform, except it early exits on NaN. - /// private static int IndexOfMinMaxCore(ReadOnlySpan x, TIndexOfMinMaxOperator op = default) where TIndexOfMinMaxOperator : struct, IIndexOfOperator { // This matches the IEEE 754:2019 `maximum`/`minimum` functions. // It propagates NaN inputs back to the caller and - // otherwise returns the greater of the inputs. + // otherwise returns the index of the greater of the inputs. // It treats +0 as greater than -0 as per the specification. int result; int i; - if (Vector.IsHardwareAccelerated && x.Length >= Vector.Count) + if (Vector.IsHardwareAccelerated && Vector.Count <= 8 && x.Length >= Vector.Count) { ref float xRef = ref MemoryMarshal.GetReference(x); - int[] resultIndexArray = new int[Vector.Count]; - int[] curIndexArray = new int[Vector.Count]; - int[] incrementArray = new int[Vector.Count]; - for (int j = 0; j < Vector.Count; j++) - { - resultIndexArray[j] = j; - curIndexArray[j] = j; - incrementArray[j] = Vector.Count; - } - - Vector resultIndex = new Vector(resultIndexArray); - Vector curIndex = new Vector(curIndexArray); - Vector increment = new Vector(incrementArray); + Vector resultIndex = new Vector([0, 1, 2, 3, 4, 5, 6, 7], 0); + Vector curIndex = resultIndex; + Vector increment = new Vector(Vector.Count); // Load the first vector as the initial set of results, and bail immediately // to scalar handling if it contains any NaNs (which don't compare equally to themselves). @@ -371,11 +357,7 @@ private static int IndexOfMinMaxCore(ReadOnlySpan // If any elements remain, handle them in one final vector. if (i != x.Length) { - for(int j = 0; j < Vector.Count; j++) - { - incrementArray[j] = x.Length - i; - } - curIndex = Vector.Add(curIndex, AsVector(ref MemoryMarshal.GetReference(incrementArray.AsSpan()), 0)); + curIndex = Vector.Add(curIndex, new Vector(x.Length - i)); current = AsVector(ref xRef, x.Length - Vector.Count); if (!Vector.EqualsAll(current, current)) From 95103d196b4f4cf7a8cf43efa2cc2b6c2579b01a Mon Sep 17 00:00:00 2001 From: Michael Sharp Date: Thu, 19 Oct 2023 09:31:33 -0600 Subject: [PATCH 14/19] minor comment changes --- .../src/System/Numerics/Tensors/TensorPrimitives.netcore.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/TensorPrimitives.netcore.cs b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/TensorPrimitives.netcore.cs index 7fcf047650b2ed..c54fc8ff26d81d 100644 --- a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/TensorPrimitives.netcore.cs +++ b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/TensorPrimitives.netcore.cs @@ -8094,7 +8094,7 @@ private interface IIndexOfOperator static virtual float IdentityValue => throw new NotSupportedException(); } - /// MathF.Max(x, y) (but NaNs may not be propagated) + /// Returns the index of MathF.Max(x, y) private readonly struct IndexOfMaxOperator : IIndexOfOperator { [MethodImpl(MethodImplOptions.AggressiveInlining)] @@ -8394,7 +8394,7 @@ public static void Invoke(ref Vector512 max, Vector512 current, re #endif } - /// MathF.Min(x, y) (but NaNs may not be propagated) + /// Returns the index of MathF.Min(x, y) private readonly struct IndexOfMinOperator : IIndexOfOperator { [MethodImpl(MethodImplOptions.AggressiveInlining)] From 10aaaee26ea006351a677973a8f7cf7daf0c71cd Mon Sep 17 00:00:00 2001 From: Michael Sharp Date: Thu, 19 Oct 2023 09:35:15 -0600 Subject: [PATCH 15/19] minor formatting fixes --- .../System/Numerics/Tensors/TensorPrimitives.netstandard.cs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/TensorPrimitives.netstandard.cs b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/TensorPrimitives.netstandard.cs index e26379c2e72b3d..95c86a0478f118 100644 --- a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/TensorPrimitives.netstandard.cs +++ b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/TensorPrimitives.netstandard.cs @@ -2439,7 +2439,7 @@ private interface IIndexOfOperator void Invoke(ref Vector result, Vector current, ref Vector resultIndex, Vector curIndex); } - /// MathF.Max(x, y) (but without guaranteed NaN propagation) + /// Returns the index of MathF.Max(x, y) private readonly struct IndexOfMaxOperator : IIndexOfOperator { [MethodImpl(MethodImplOptions.AggressiveInlining)] @@ -2496,6 +2496,7 @@ public int Invoke(Vector result, Vector resultIndex) curIn = resultIndex[i]; } } + return curIn; } @@ -2580,6 +2581,7 @@ public int Invoke(Vector result, Vector maxIndex) curIn = maxIndex[i]; } } + return curIn; } @@ -2662,6 +2664,7 @@ public int Invoke(Vector result, Vector resultIndex) curIn = resultIndex[i]; } } + return curIn; } @@ -2746,6 +2749,7 @@ public int Invoke(Vector result, Vector resultIndex) curIn = resultIndex[i]; } } + return curIn; } From e9c5534fd72bb503d965727a5009f3d9e961c98b Mon Sep 17 00:00:00 2001 From: Michael Sharp Date: Thu, 19 Oct 2023 10:00:10 -0600 Subject: [PATCH 16/19] added inlining --- .../src/System/Numerics/Tensors/TensorPrimitives.netcore.cs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/TensorPrimitives.netcore.cs b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/TensorPrimitives.netcore.cs index c54fc8ff26d81d..b4d1ed5cb53831 100644 --- a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/TensorPrimitives.netcore.cs +++ b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/TensorPrimitives.netcore.cs @@ -10103,6 +10103,7 @@ public static Vector512 Invoke(Vector512 x) #endif } + [MethodImpl(MethodImplOptions.AggressiveInlining)] private static Vector128 ElementWiseSelect(Vector128 mask, Vector128 left, Vector128 right) { if (Sse41.IsSupported) @@ -10112,6 +10113,7 @@ private static Vector128 ElementWiseSelect(Vector128 mask, Vector1 else return Vector128.ConditionalSelect(mask, left, right); } + [MethodImpl(MethodImplOptions.AggressiveInlining)] private static Vector128 ElementWiseSelect(Vector128 mask, Vector128 left, Vector128 right) { if (Sse41.IsSupported) @@ -10121,6 +10123,7 @@ private static Vector128 ElementWiseSelect(Vector128 mask, Vector128 ElementWiseSelect(Vector256 mask, Vector256 left, Vector256 right) { if (Avx2.IsSupported) @@ -10130,6 +10133,7 @@ private static Vector256 ElementWiseSelect(Vector256 mask, Vector2 else return Vector256.ConditionalSelect(mask, left, right); } + [MethodImpl(MethodImplOptions.AggressiveInlining)] private static Vector256 ElementWiseSelect(Vector256 mask, Vector256 left, Vector256 right) { if (Avx2.IsSupported) @@ -10140,6 +10144,7 @@ private static Vector256 ElementWiseSelect(Vector256 mask, Vector256 ElementWiseSelect(Vector512 mask, Vector512 left, Vector512 right) { if (Avx512F.IsSupported) @@ -10149,6 +10154,7 @@ private static Vector512 ElementWiseSelect(Vector512 mask, Vector5 else return Vector512.ConditionalSelect(mask, left, right); } + [MethodImpl(MethodImplOptions.AggressiveInlining)] private static Vector512 ElementWiseSelect(Vector512 mask, Vector512 left, Vector512 right) { if (Avx512F.IsSupported) From f92123a9347da8c17002d5ec1aff810c3cf2dbb0 Mon Sep 17 00:00:00 2001 From: Michael Sharp Date: Thu, 19 Oct 2023 11:27:16 -0600 Subject: [PATCH 17/19] fixes from PR comments --- .../Tensors/TensorPrimitives.netcore.cs | 415 +++++++----------- .../Tensors/TensorPrimitives.netstandard.cs | 204 ++++----- 2 files changed, 232 insertions(+), 387 deletions(-) diff --git a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/TensorPrimitives.netcore.cs b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/TensorPrimitives.netcore.cs index b4d1ed5cb53831..ed9bae090ebb4c 100644 --- a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/TensorPrimitives.netcore.cs +++ b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/TensorPrimitives.netcore.cs @@ -1225,15 +1225,11 @@ private static float MinMaxCore(ReadOnlySpan x) } } - /// - /// This is the same as , - /// except it early exits on NaN. - /// private static int IndexOfMinMaxCore(ReadOnlySpan x) where TIndexOfMinMax : struct, IIndexOfOperator { // This matches the IEEE 754:2019 `maximum`/`minimum` functions. // It propagates NaN inputs back to the caller and - // otherwise returns the greater of the inputs. + // otherwise returns the index of the greater of the inputs. // It treats +0 as greater than -0 as per the specification. #if NET8_0_OR_GREATER @@ -1241,15 +1237,15 @@ private static int IndexOfMinMaxCore(ReadOnlySpan x) wher { ref float xRef = ref MemoryMarshal.GetReference(x); Vector512 resultIndex = Vector512.Create(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); - Vector512 curIndex = Vector512.Create(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + Vector512 curIndex = resultIndex; Vector512 increment = Vector512.Create(Vector512.Count); // Load the first vector as the initial set of results, and bail immediately // to scalar handling if it contains any NaNs (which don't compare equally to themselves). Vector512 result = Vector512.LoadUnsafe(ref xRef); Vector512 current; - Vector512 nanMask = ~Vector512.Equals(result, result); + Vector512 nanMask = ~Vector512.Equals(result, result); if (nanMask != Vector512.Zero) { return IndexOfFirstMatch(nanMask); @@ -1266,13 +1262,13 @@ private static int IndexOfMinMaxCore(ReadOnlySpan x) wher curIndex += increment; nanMask = ~Vector512.Equals(current, current); - if (nanMask != Vector512.Zero) { return i + IndexOfFirstMatch(nanMask); } TIndexOfMinMax.Invoke(ref result, current, ref resultIndex, curIndex); + i += Vector512.Count; } @@ -1283,7 +1279,6 @@ private static int IndexOfMinMaxCore(ReadOnlySpan x) wher curIndex += Vector512.Create(x.Length - i); nanMask = ~Vector512.Equals(current, current); - if (nanMask != Vector512.Zero) { return curIndex[IndexOfFirstMatch(nanMask)]; @@ -1293,7 +1288,6 @@ private static int IndexOfMinMaxCore(ReadOnlySpan x) wher } // Aggregate the lanes in the vector to create the final scalar result. - return TIndexOfMinMax.Invoke(result, resultIndex); } #endif @@ -1302,15 +1296,15 @@ private static int IndexOfMinMaxCore(ReadOnlySpan x) wher { ref float xRef = ref MemoryMarshal.GetReference(x); Vector256 resultIndex = Vector256.Create(0, 1, 2, 3, 4, 5, 6, 7); - Vector256 curIndex = Vector256.Create(0, 1, 2, 3, 4, 5, 6, 7); + Vector256 curIndex = resultIndex; Vector256 increment = Vector256.Create(Vector256.Count); // Load the first vector as the initial set of results, and bail immediately // to scalar handling if it contains any NaNs (which don't compare equally to themselves). Vector256 result = Vector256.LoadUnsafe(ref xRef); Vector256 current; - Vector256 nanMask = ~Vector256.Equals(result, result); + Vector256 nanMask = ~Vector256.Equals(result, result); if (nanMask != Vector256.Zero) { return IndexOfFirstMatch(nanMask); @@ -1327,7 +1321,6 @@ private static int IndexOfMinMaxCore(ReadOnlySpan x) wher curIndex += increment; nanMask = ~Vector256.Equals(current, current); - if (nanMask != Vector256.Zero) { return i + IndexOfFirstMatch(nanMask); @@ -1345,14 +1338,12 @@ private static int IndexOfMinMaxCore(ReadOnlySpan x) wher curIndex += Vector256.Create(x.Length - i); nanMask = ~Vector256.Equals(current, current); - if (nanMask != Vector256.Zero) { return curIndex[IndexOfFirstMatch(nanMask)]; } TIndexOfMinMax.Invoke(ref result, current, ref resultIndex, curIndex); - } // Aggregate the lanes in the vector to create the final scalar result. @@ -1363,15 +1354,15 @@ private static int IndexOfMinMaxCore(ReadOnlySpan x) wher { ref float xRef = ref MemoryMarshal.GetReference(x); Vector128 resultIndex = Vector128.Create(0, 1, 2, 3); - Vector128 curIndex = Vector128.Create(0, 1, 2, 3); + Vector128 curIndex = resultIndex; Vector128 increment = Vector128.Create(Vector128.Count); // Load the first vector as the initial set of results, and bail immediately // to scalar handling if it contains any NaNs (which don't compare equally to themselves). Vector128 result = Vector128.LoadUnsafe(ref xRef); Vector128 current; - Vector128 nanMask = ~Vector128.Equals(result, result); + Vector128 nanMask = ~Vector128.Equals(result, result); if (nanMask != Vector128.Zero) { return IndexOfFirstMatch(nanMask); @@ -1388,7 +1379,6 @@ private static int IndexOfMinMaxCore(ReadOnlySpan x) wher curIndex += increment; nanMask = ~Vector128.Equals(current, current); - if (nanMask != Vector128.Zero) { return i + IndexOfFirstMatch(nanMask); @@ -1405,15 +1395,14 @@ private static int IndexOfMinMaxCore(ReadOnlySpan x) wher curIndex += Vector128.Create(x.Length - i); current = Vector128.LoadUnsafe(ref xRef, (uint)(x.Length - Vector128.Count)); - nanMask = ~Vector128.Equals(current, current); + nanMask = ~Vector128.Equals(current, current); if (nanMask != Vector128.Zero) { return curIndex[IndexOfFirstMatch(nanMask)]; } TIndexOfMinMax.Invoke(ref result, current, ref resultIndex, curIndex); - } // Aggregate the lanes in the vector to create the final scalar result. @@ -1421,7 +1410,26 @@ private static int IndexOfMinMaxCore(ReadOnlySpan x) wher } // Scalar path used when either vectorization is not supported or the input is too small to vectorize. - return TIndexOfMinMax.Invoke(x); + float curResult = x[0]; + int curIn = 0; + if (float.IsNaN(curResult)) + { + return curIn; + } + + for (int i = 1; i < x.Length; i++) + { + float current = x[i]; + //float currentAbs = MathF.Abs(current); + if (float.IsNaN(current)) + { + return i; + } + + curIn = TIndexOfMinMax.Invoke(ref curResult, current, curIn, i); + } + + return curIn; } #if NET8_0_OR_GREATER @@ -8081,7 +8089,7 @@ public static Vector512 Invoke(Vector512 x, Vector512 y) => private interface IIndexOfOperator { - static abstract int Invoke(ReadOnlySpan result); + static abstract int Invoke(ref float result, float current, int resultIndex, int curIndex); static abstract int Invoke(Vector128 result, Vector128 resultIndex); static abstract void Invoke(ref Vector128 result, Vector128 current, ref Vector128 resultIndex, Vector128 curIndex); static abstract int Invoke(Vector256 result, Vector256 resultIndex); @@ -8090,61 +8098,23 @@ private interface IIndexOfOperator static abstract int Invoke(Vector512 result, Vector512 resultIndex); static abstract void Invoke(ref Vector512 result, Vector512 current, ref Vector512 resultIndex, Vector512 curIndex); #endif - - static virtual float IdentityValue => throw new NotSupportedException(); } /// Returns the index of MathF.Max(x, y) private readonly struct IndexOfMaxOperator : IIndexOfOperator { - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static int Invoke(ReadOnlySpan result) - { - float curMax = result[0]; - int curIn = 0; - if (float.IsNaN(curMax)) - { - return curIn; - } - - for (int i = 1; i < result.Length; i++) - { - float current = result[i]; - if (float.IsNaN(current)) - { - return i; - } - - if (curMax == current) - { - if (IsNegative(curMax) && !IsNegative(current)) - { - curMax = current; - curIn = i; - } - } - else if (current > curMax) - { - curMax = current; - curIn = i; - } - } - - return curIn; - } - [MethodImpl(MethodImplOptions.AggressiveInlining)] public static int Invoke(Vector128 result, Vector128 maxIndex) { - Vector128 tmp128 = Vector128.Shuffle(result, Vector128.Create(2, 3, 0, 1)); - Vector128 tmpIndex128 = Vector128.Shuffle(maxIndex, Vector128.Create(2, 3, 0, 1)); + Vector128 tmpResult = Vector128.Shuffle(result, Vector128.Create(2, 3, 0, 1)); + Vector128 tmpIndex = Vector128.Shuffle(maxIndex, Vector128.Create(2, 3, 0, 1)); - Invoke(ref result, tmp128, ref maxIndex, tmpIndex128); + Invoke(ref result, tmpResult, ref maxIndex, tmpIndex); - tmp128 = Vector128.Shuffle(result, Vector128.Create(1, 0, 3, 2)); - tmpIndex128 = Vector128.Shuffle(maxIndex, Vector128.Create(1, 0, 3, 2)); + tmpResult = Vector128.Shuffle(result, Vector128.Create(1, 0, 3, 2)); + tmpIndex = Vector128.Shuffle(maxIndex, Vector128.Create(1, 0, 3, 2)); - Invoke(ref result, tmp128, ref maxIndex, tmpIndex128); + Invoke(ref result, tmpResult, ref maxIndex, tmpIndex); return maxIndex.ToScalar(); } @@ -8154,8 +8124,7 @@ public static void Invoke(ref Vector128 max, Vector128 current, re Vector128 greaterThanMask = Vector128.GreaterThan(max, current); Vector128 equalMask = Vector128.Equals(max, current); - - if (equalMask.AsInt32() != Vector128.Zero) + if (equalMask.AsInt32() != Vector128.Zero) { Vector128 negativeMask = IsNegative(current); Vector128 lessThanMask = Vector128.LessThan(maxIndex, curIndex); @@ -8185,7 +8154,6 @@ public static void Invoke(ref Vector256 max, Vector256 current, re Vector256 greaterThanMask = Vector256.GreaterThan(max, current); Vector256 equalMask = Vector256.Equals(max, current); - if (equalMask.AsInt32() != Vector256.Zero) { Vector256 negativeMask = IsNegative(current); @@ -8201,21 +8169,14 @@ public static void Invoke(ref Vector256 max, Vector256 current, re #if NET8_0_OR_GREATER [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static int Invoke(Vector512 result, Vector512 maxIndex) + public static int Invoke(Vector512 result, Vector512 resultIndex) { - // Max the upper/lower halves of the Vector512 - Vector256 resultLower256 = result.GetLower(); - Vector256 indexLower256 = maxIndex.GetLower(); - - Invoke(ref resultLower256, result.GetUpper(), ref indexLower256, maxIndex.GetUpper()); - - // Max the upper/lower halves of the Vector256 - Vector128 resultLower128 = resultLower256.GetLower(); - Vector128 indexLower128 = indexLower256.GetLower(); - - Invoke(ref resultLower128, resultLower256.GetUpper(), ref indexLower128, indexLower256.GetUpper()); + // Min the upper/lower halves of the Vector512 + Vector256 resultLower = result.GetLower(); + Vector256 indexLower = resultIndex.GetLower(); - return Invoke(resultLower128, indexLower128); + Invoke(ref resultLower, result.GetUpper(), ref indexLower, resultIndex.GetUpper()); + return Invoke(resultLower, indexLower); } [MethodImpl(MethodImplOptions.AggressiveInlining)] @@ -8224,7 +8185,6 @@ public static void Invoke(ref Vector512 max, Vector512 current, re Vector512 greaterThanMask = Vector512.GreaterThan(max, current); Vector512 equalMask = Vector512.Equals(max, current); - if (equalMask.AsInt32() != Vector512.Zero) { Vector512 negativeMask = IsNegative(current); @@ -8238,62 +8198,42 @@ public static void Invoke(ref Vector512 max, Vector512 current, re maxIndex = ElementWiseSelect(greaterThanMask.AsInt32(), maxIndex, curIndex); } #endif - } - private readonly struct IndexOfMaxMagnitudeOperator : IIndexOfOperator - { [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static int Invoke(ReadOnlySpan result) + public static int Invoke(ref float result, float current, int resultIndex, int curIndex) { - float curMax = result[0]; - float curMaxAbs = MathF.Abs(curMax); - int curIn = 0; - if (float.IsNaN(curMax)) - { - return curIn; - } - - for (int i = 1; i < result.Length; i++) + if (result == current) { - float current = result[i]; - float currentAbs = MathF.Abs(current); - if (float.IsNaN(current)) - { - return i; - } - - if (curMaxAbs == currentAbs) + if (IsNegative(result) && !IsNegative(current)) { - if (IsNegative(curMax) && !IsNegative(current)) - { - curMax = current; - curMaxAbs = MathF.Abs(current); - curIn = i; - } - } - else if (currentAbs > curMaxAbs) - { - curMax = current; - curMaxAbs = MathF.Abs(current); - curIn = i; + result = current; + return curIndex; } } + else if (current > result) + { + result = current; + return curIndex; + } - return curIn; + return resultIndex; } + } + private readonly struct IndexOfMaxMagnitudeOperator : IIndexOfOperator + { [MethodImpl(MethodImplOptions.AggressiveInlining)] public static int Invoke(Vector128 result, Vector128 maxIndex) { - Vector128 tmp128 = Vector128.Shuffle(result, Vector128.Create(2, 3, 0, 1)); - Vector128 tmpIndex128 = Vector128.Shuffle(maxIndex, Vector128.Create(2, 3, 0, 1)); + Vector128 tmpResult = Vector128.Shuffle(result, Vector128.Create(2, 3, 0, 1)); + Vector128 tmpIndex = Vector128.Shuffle(maxIndex, Vector128.Create(2, 3, 0, 1)); - Invoke(ref result, tmp128, ref maxIndex, tmpIndex128); + Invoke(ref result, tmpResult, ref maxIndex, tmpIndex); - tmp128 = Vector128.Shuffle(result, Vector128.Create(1, 0, 3, 2)); - tmpIndex128 = Vector128.Shuffle(maxIndex, Vector128.Create(1, 0, 3, 2)); + tmpResult = Vector128.Shuffle(result, Vector128.Create(1, 0, 3, 2)); + tmpIndex = Vector128.Shuffle(maxIndex, Vector128.Create(1, 0, 3, 2)); - Invoke(ref result, tmp128, ref maxIndex, tmpIndex128); + Invoke(ref result, tmpResult, ref maxIndex, tmpIndex); return maxIndex.ToScalar(); } @@ -8305,7 +8245,6 @@ public static void Invoke(ref Vector128 max, Vector128 current, re Vector128 greaterThanMask = Vector128.GreaterThan(maxMag, currentMag); Vector128 equalMask = Vector128.Equals(max, current); - if (equalMask.AsInt32() != Vector128.Zero) { Vector128 negativeMask = IsNegative(current); @@ -8338,7 +8277,6 @@ public static void Invoke(ref Vector256 max, Vector256 current, re Vector256 greaterThanMask = Vector256.GreaterThan(maxMag, currentMag); Vector256 equalMask = Vector256.Equals(max, current); - if (equalMask.AsInt32() != Vector256.Zero) { Vector256 negativeMask = IsNegative(current); @@ -8354,21 +8292,14 @@ public static void Invoke(ref Vector256 max, Vector256 current, re #if NET8_0_OR_GREATER [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static int Invoke(Vector512 result, Vector512 maxIndex) + public static int Invoke(Vector512 result, Vector512 resultIndex) { - // Max the upper/lower halves of the Vector512 - Vector256 resultLower256 = result.GetLower(); - Vector256 indexLower256 = maxIndex.GetLower(); - - Invoke(ref resultLower256, result.GetUpper(), ref indexLower256, maxIndex.GetUpper()); - - // Max the upper/lower halves of the Vector256 - Vector128 resultLower128 = resultLower256.GetLower(); - Vector128 indexLower128 = indexLower256.GetLower(); - - Invoke(ref resultLower128, resultLower256.GetUpper(), ref indexLower128, indexLower256.GetUpper()); + // Min the upper/lower halves of the Vector512 + Vector256 resultLower = result.GetLower(); + Vector256 indexLower = resultIndex.GetLower(); - return Invoke(resultLower128, indexLower128); + Invoke(ref resultLower, result.GetUpper(), ref indexLower, resultIndex.GetUpper()); + return Invoke(resultLower, indexLower); } [MethodImpl(MethodImplOptions.AggressiveInlining)] @@ -8378,7 +8309,6 @@ public static void Invoke(ref Vector512 max, Vector512 current, re Vector512 greaterThanMask = Vector512.GreaterThan(maxMag, currentMag); Vector512 equalMask = Vector512.Equals(max, current); - if (equalMask.AsInt32() != Vector512.Zero) { Vector512 negativeMask = IsNegative(current); @@ -8392,59 +8322,46 @@ public static void Invoke(ref Vector512 max, Vector512 current, re maxIndex = ElementWiseSelect(greaterThanMask.AsInt32(), maxIndex, curIndex); } #endif - } - /// Returns the index of MathF.Min(x, y) - private readonly struct IndexOfMinOperator : IIndexOfOperator - { [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static int Invoke(ReadOnlySpan result) + public static int Invoke(ref float result, float current, int resultIndex, int curIndex) { - float curMin = result[0]; - int curIn = 0; - if (float.IsNaN(curMin)) - { - return curIn; - } + float curMaxAbs = MathF.Abs(result); + float currentAbs = MathF.Abs(current); - for (int i = 1; i < result.Length; i++) + if (curMaxAbs == currentAbs) { - float current = result[i]; - if (float.IsNaN(current)) + if (IsNegative(result) && !IsNegative(current)) { - return i; - } - - if (curMin == current) - { - if (IsPositive(curMin) && !IsPositive(current)) - { - curMin = current; - curIn = i; - } - } - else if (current < curMin) - { - curMin = current; - curIn = i; + result = current; + return curIndex; } } + else if (currentAbs > curMaxAbs) + { + result = current; + return curIndex; + } - return curIn; + return resultIndex; } + } + /// Returns the index of MathF.Min(x, y) + private readonly struct IndexOfMinOperator : IIndexOfOperator + { [MethodImpl(MethodImplOptions.AggressiveInlining)] public static int Invoke(Vector128 result, Vector128 resultIndex) { - Vector128 tmp128 = Vector128.Shuffle(result, Vector128.Create(2, 3, 0, 1)); - Vector128 tmpIndex128 = Vector128.Shuffle(resultIndex, Vector128.Create(2, 3, 0, 1)); + Vector128 tmpResult = Vector128.Shuffle(result, Vector128.Create(2, 3, 0, 1)); + Vector128 tmpIndex = Vector128.Shuffle(resultIndex, Vector128.Create(2, 3, 0, 1)); - Invoke(ref result, tmp128, ref resultIndex, tmpIndex128); + Invoke(ref result, tmpResult, ref resultIndex, tmpIndex); - tmp128 = Vector128.Shuffle(result, Vector128.Create(1, 0, 3, 2)); - tmpIndex128 = Vector128.Shuffle(resultIndex, Vector128.Create(1, 0, 3, 2)); + tmpResult = Vector128.Shuffle(result, Vector128.Create(1, 0, 3, 2)); + tmpIndex = Vector128.Shuffle(resultIndex, Vector128.Create(1, 0, 3, 2)); - Invoke(ref result, tmp128, ref resultIndex, tmpIndex128); + Invoke(ref result, tmpResult, ref resultIndex, tmpIndex); return resultIndex.ToScalar(); } @@ -8454,7 +8371,6 @@ public static void Invoke(ref Vector128 result, Vector128 current, Vector128 lessThanMask = Vector128.LessThan(result, current); Vector128 equalMask = Vector128.Equals(result, current); - if (equalMask.AsInt32() != Vector128.Zero) { Vector128 negativeMask = IsNegative(current); @@ -8486,7 +8402,6 @@ public static void Invoke(ref Vector256 result, Vector256 current, Vector256 lessThanMask = Vector256.LessThan(result, current); Vector256 equalMask = Vector256.Equals(result, current); - if (equalMask.AsInt32() != Vector256.Zero) { Vector256 negativeMask = IsNegative(current); @@ -8505,18 +8420,11 @@ public static void Invoke(ref Vector256 result, Vector256 current, public static int Invoke(Vector512 result, Vector512 resultIndex) { // Min the upper/lower halves of the Vector512 - Vector256 resultLower256 = result.GetLower(); - Vector256 indexLower256 = resultIndex.GetLower(); - - Invoke(ref resultLower256, result.GetUpper(), ref indexLower256, resultIndex.GetUpper()); - - // Min the upper/lower halves of the Vector256 - Vector128 resultLower128 = resultLower256.GetLower(); - Vector128 indexLower128 = indexLower256.GetLower(); - - Invoke(ref resultLower128, resultLower256.GetUpper(), ref indexLower128, indexLower256.GetUpper()); + Vector256 resultLower = result.GetLower(); + Vector256 indexLower = resultIndex.GetLower(); - return Invoke(resultLower128, indexLower128); + Invoke(ref resultLower, result.GetUpper(), ref indexLower, resultIndex.GetUpper()); + return Invoke(resultLower, indexLower); } [MethodImpl(MethodImplOptions.AggressiveInlining)] @@ -8525,7 +8433,6 @@ public static void Invoke(ref Vector512 result, Vector512 current, Vector512 lessThanMask = Vector512.LessThan(result, current); Vector512 equalMask = Vector512.Equals(result, current); - if (equalMask.AsInt32() != Vector512.Zero) { Vector512 negativeMask = IsNegative(current); @@ -8539,62 +8446,42 @@ public static void Invoke(ref Vector512 result, Vector512 current, resultIndex = ElementWiseSelect(lessThanMask.AsInt32(), resultIndex, curIndex); } #endif - } - private readonly struct IndexOfMinMagnitudeOperator : IIndexOfOperator - { [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static int Invoke(ReadOnlySpan result) + public static int Invoke(ref float result, float current, int resultIndex, int curIndex) { - float curMin = result[0]; - float curMinAbs = MathF.Abs(curMin); - int curIn = 0; - if (float.IsNaN(curMin)) - { - return curIn; - } - - for (int i = 1; i < result.Length; i++) + if (result == current) { - float current = result[i]; - float currentAbs = MathF.Abs(current); - if (float.IsNaN(current)) - { - return i; - } - - if (curMinAbs == currentAbs) - { - if (IsPositive(curMin) && !IsPositive(current)) - { - curMin = current; - curMinAbs = MathF.Abs(current); - curIn = i; - } - } - else if (currentAbs < curMinAbs) + if (IsPositive(result) && !IsPositive(current)) { - curMin = current; - curMinAbs = MathF.Abs(current); - curIn = i; + result = current; + return curIndex; } } + else if (current < result) + { + result = current; + return curIndex; + } - return curIn; + return resultIndex; } + } + private readonly struct IndexOfMinMagnitudeOperator : IIndexOfOperator + { [MethodImpl(MethodImplOptions.AggressiveInlining)] public static int Invoke(Vector128 result, Vector128 resultIndex) { - Vector128 tmp128 = Vector128.Shuffle(result, Vector128.Create(2, 3, 0, 1)); - Vector128 tmpIndex128 = Vector128.Shuffle(resultIndex, Vector128.Create(2, 3, 0, 1)); + Vector128 tmpResult = Vector128.Shuffle(result, Vector128.Create(2, 3, 0, 1)); + Vector128 tmpIndex = Vector128.Shuffle(resultIndex, Vector128.Create(2, 3, 0, 1)); - Invoke(ref result, tmp128, ref resultIndex, tmpIndex128); + Invoke(ref result, tmpResult, ref resultIndex, tmpIndex); - tmp128 = Vector128.Shuffle(result, Vector128.Create(1, 0, 3, 2)); - tmpIndex128 = Vector128.Shuffle(resultIndex, Vector128.Create(1, 0, 3, 2)); + tmpResult = Vector128.Shuffle(result, Vector128.Create(1, 0, 3, 2)); + tmpIndex = Vector128.Shuffle(resultIndex, Vector128.Create(1, 0, 3, 2)); - Invoke(ref result, tmp128, ref resultIndex, tmpIndex128); + Invoke(ref result, tmpResult, ref resultIndex, tmpIndex); return resultIndex.ToScalar(); } @@ -8606,7 +8493,6 @@ public static void Invoke(ref Vector128 result, Vector128 current, Vector128 lessThanMask = Vector128.LessThan(minMag, currentMag); Vector128 equalMask = Vector128.Equals(result, current); - if (equalMask.AsInt32() != Vector128.Zero) { Vector128 negativeMask = IsNegative(current); @@ -8639,7 +8525,6 @@ public static void Invoke(ref Vector256 result, Vector256 current, Vector256 lessThanMask = Vector256.LessThan(minMag, currentMag); Vector256 equalMask = Vector256.Equals(result, current); - if (equalMask.AsInt32() != Vector256.Zero) { Vector256 negativeMask = IsNegative(current); @@ -8658,18 +8543,11 @@ public static void Invoke(ref Vector256 result, Vector256 current, public static int Invoke(Vector512 result, Vector512 resultIndex) { // Min the upper/lower halves of the Vector512 - Vector256 resultLower256 = result.GetLower(); - Vector256 indexLower256 = resultIndex.GetLower(); - - Invoke(ref resultLower256, result.GetUpper(), ref indexLower256, resultIndex.GetUpper()); + Vector256 resultLower = result.GetLower(); + Vector256 indexLower = resultIndex.GetLower(); - // Min the upper/lower halves of the Vector256 - Vector128 resultLower128 = resultLower256.GetLower(); - Vector128 indexLower128 = indexLower256.GetLower(); - - Invoke(ref resultLower128, resultLower256.GetUpper(), ref indexLower128, indexLower256.GetUpper()); - - return Invoke(resultLower128, indexLower128); + Invoke(ref resultLower, result.GetUpper(), ref indexLower, resultIndex.GetUpper()); + return Invoke(resultLower, indexLower); } [MethodImpl(MethodImplOptions.AggressiveInlining)] @@ -8680,7 +8558,6 @@ public static void Invoke(ref Vector512 result, Vector512 current, Vector512 lessThanMask = Vector512.LessThan(minMag, currentMag); Vector512 equalMask = Vector512.Equals(result, current); - if (equalMask.AsInt32() != Vector512.Zero) { Vector512 negativeMask = IsNegative(current); @@ -8694,6 +8571,28 @@ public static void Invoke(ref Vector512 result, Vector512 current, resultIndex = ElementWiseSelect(lessThanMask.AsInt32(), resultIndex, curIndex); } #endif + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static int Invoke(ref float result, float current, int resultIndex, int curIndex) + { + float curMinAbs = MathF.Abs(result); + float currentAbs = MathF.Abs(current); + if (curMinAbs == currentAbs) + { + if (IsPositive(result) && !IsPositive(current)) + { + result = current; + return curIndex; + } + } + else if (currentAbs < curMinAbs) + { + result = current; + return curIndex; + } + + return resultIndex; + } } /// MathF.Max(x, y) @@ -10107,40 +10006,36 @@ public static Vector512 Invoke(Vector512 x) private static Vector128 ElementWiseSelect(Vector128 mask, Vector128 left, Vector128 right) { if (Sse41.IsSupported) - { return Sse41.BlendVariable(left, right, ~mask); - } - else return Vector128.ConditionalSelect(mask, left, right); + + return Vector128.ConditionalSelect(mask, left, right); } [MethodImpl(MethodImplOptions.AggressiveInlining)] private static Vector128 ElementWiseSelect(Vector128 mask, Vector128 left, Vector128 right) { if (Sse41.IsSupported) - { return Sse41.BlendVariable(left, right, ~mask); - } - else return Vector128.ConditionalSelect(mask, left, right); + + return Vector128.ConditionalSelect(mask, left, right); } [MethodImpl(MethodImplOptions.AggressiveInlining)] private static Vector256 ElementWiseSelect(Vector256 mask, Vector256 left, Vector256 right) { if (Avx2.IsSupported) - { return Avx2.BlendVariable(left, right, ~mask); - } - else return Vector256.ConditionalSelect(mask, left, right); + + return Vector256.ConditionalSelect(mask, left, right); } [MethodImpl(MethodImplOptions.AggressiveInlining)] private static Vector256 ElementWiseSelect(Vector256 mask, Vector256 left, Vector256 right) { if (Avx2.IsSupported) - { return Avx2.BlendVariable(left, right, ~mask); - } - else return Vector256.ConditionalSelect(mask, left, right); + + return Vector256.ConditionalSelect(mask, left, right); } #if NET8_0_OR_GREATER @@ -10148,20 +10043,18 @@ private static Vector256 ElementWiseSelect(Vector256 mask, Vector256 ElementWiseSelect(Vector512 mask, Vector512 left, Vector512 right) { if (Avx512F.IsSupported) - { return Avx512F.BlendVariable(left, right, ~mask); - } - else return Vector512.ConditionalSelect(mask, left, right); + + return Vector512.ConditionalSelect(mask, left, right); } [MethodImpl(MethodImplOptions.AggressiveInlining)] private static Vector512 ElementWiseSelect(Vector512 mask, Vector512 left, Vector512 right) { if (Avx512F.IsSupported) - { return Avx512F.BlendVariable(left, right, ~mask); - } - else return Vector512.ConditionalSelect(mask, left, right); + + return Vector512.ConditionalSelect(mask, left, right); } #endif diff --git a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/TensorPrimitives.netstandard.cs b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/TensorPrimitives.netstandard.cs index 95c86a0478f118..4de2e2968aeb69 100644 --- a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/TensorPrimitives.netstandard.cs +++ b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/TensorPrimitives.netstandard.cs @@ -377,7 +377,26 @@ private static int IndexOfMinMaxCore(ReadOnlySpan // Scalar path used when either vectorization is not supported, the input is too small to vectorize, // or a NaN is encountered. Scalar: - return op.Invoke(x); + float curResult = x[0]; + int curIn = 0; + if (float.IsNaN(curResult)) + { + return curIn; + } + + for (i = 1; i < x.Length; i++) + { + float current = x[i]; + //float currentAbs = MathF.Abs(current); + if (float.IsNaN(current)) + { + return i; + } + + curIn = op.Invoke(ref curResult, current, curIn, i); + } + + return curIn; } /// Performs an element-wise operation on and writes the results to . @@ -2434,7 +2453,7 @@ public Vector Invoke(Vector x, Vector y) private interface IIndexOfOperator { - int Invoke(ReadOnlySpan result); + int Invoke(ref float result, float current, int resultIndex, int curIndex); int Invoke(Vector result, Vector resultIndex); void Invoke(ref Vector result, Vector current, ref Vector resultIndex, Vector curIndex); } @@ -2442,42 +2461,6 @@ private interface IIndexOfOperator /// Returns the index of MathF.Max(x, y) private readonly struct IndexOfMaxOperator : IIndexOfOperator { - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public int Invoke(ReadOnlySpan result) - { - float curMax = result[0]; - int curIn = 0; - if (float.IsNaN(curMax)) - { - return curIn; - } - - for (int i = 1; i < result.Length; i++) - { - float current = result[i]; - if (float.IsNaN(current)) - { - return i; - } - - if (curMax == current) - { - if (IsNegative(curMax) && !IsNegative(current)) - { - curMax = current; - curIn = i; - } - } - else if (current > curMax) - { - curMax = current; - curIn = i; - } - } - - return curIn; - } - [MethodImpl(MethodImplOptions.AggressiveInlining)] public int Invoke(Vector result, Vector resultIndex) { @@ -2519,48 +2502,51 @@ public void Invoke(ref Vector result, Vector current, ref Vector result) + { + result = current; + return curIndex; + } + + return resultIndex; + } } private readonly struct IndexOfMaxMagnitudeOperator : IIndexOfOperator { [MethodImpl(MethodImplOptions.AggressiveInlining)] - public int Invoke(ReadOnlySpan result) + public int Invoke(ref float result, float current, int resultIndex, int curIndex) { - float curMax = result[0]; - float curMaxAbs = MathF.Abs(curMax); - int curIn = 0; - if (float.IsNaN(curMax)) - { - return curIn; - } + float curMaxAbs = MathF.Abs(result); + float currentAbs = MathF.Abs(current); - for (int i = 1; i < result.Length; i++) + if (curMaxAbs == currentAbs) { - float current = result[i]; - float currentAbs = MathF.Abs(current); - if (float.IsNaN(current)) + if (IsNegative(result) && !IsNegative(current)) { - return i; - } - - if (curMaxAbs == currentAbs) - { - if (IsNegative(curMax) && !IsNegative(current)) - { - curMax = current; - curMaxAbs = MathF.Abs(current); - curIn = i; - } - } - else if (currentAbs > curMaxAbs) - { - curMax = current; - curMaxAbs = MathF.Abs(current); - curIn = i; + result = current; + return curIndex; } } + else if (currentAbs > curMaxAbs) + { + result = current; + return curIndex; + } - return curIn; + return resultIndex; } [MethodImpl(MethodImplOptions.AggressiveInlining)] @@ -2611,39 +2597,23 @@ public void Invoke(ref Vector result, Vector current, ref Vector result) + public int Invoke(ref float result, float current, int resultIndex, int curIndex) { - float curMin = result[0]; - int curIn = 0; - if (float.IsNaN(curMin)) + if (result == current) { - return curIn; - } - - for (int i = 1; i < result.Length; i++) - { - float current = result[i]; - if (float.IsNaN(current)) - { - return i; - } - - if (curMin == current) - { - if (IsPositive(curMin) && !IsPositive(current)) - { - curMin = current; - curIn = i; - } - } - else if (current < curMin) + if (IsPositive(result) && !IsPositive(current)) { - curMin = current; - curIn = i; + result = current; + return curIndex; } } + else if (current < result) + { + result = current; + return curIndex; + } - return curIn; + return resultIndex; } [MethodImpl(MethodImplOptions.AggressiveInlining)] @@ -2692,43 +2662,25 @@ public void Invoke(ref Vector result, Vector current, ref Vector result) + public int Invoke(ref float result, float current, int resultIndex, int curIndex) { - float curMin = result[0]; - float curMinAbs = MathF.Abs(curMin); - int curIn = 0; - if (float.IsNaN(curMin)) - { - return curIn; - } - - for (int i = 1; i < result.Length; i++) + float curMinAbs = MathF.Abs(result); + float currentAbs = MathF.Abs(current); + if (curMinAbs == currentAbs) { - float current = result[i]; - float currentAbs = MathF.Abs(current); - if (float.IsNaN(current)) + if (IsPositive(result) && !IsPositive(current)) { - return i; - } - - if (curMinAbs == currentAbs) - { - if (IsPositive(curMin) && !IsPositive(current)) - { - curMin = current; - curMinAbs = MathF.Abs(current); - curIn = i; - } - } - else if (currentAbs < curMinAbs) - { - curMin = current; - curMinAbs = MathF.Abs(current); - curIn = i; + result = current; + return curIndex; } } + else if (currentAbs < curMinAbs) + { + result = current; + return curIndex; + } - return curIn; + return resultIndex; } [MethodImpl(MethodImplOptions.AggressiveInlining)] From 13f756a5befc2b488318dd88243dea8d5e7e03f3 Mon Sep 17 00:00:00 2001 From: Michael Sharp Date: Fri, 20 Oct 2023 00:53:49 -0600 Subject: [PATCH 18/19] comments from pr --- .../Numerics/Tensors/TensorPrimitives.netcore.cs | 12 +++++++----- .../Tensors/TensorPrimitives.netstandard.cs | 13 +++++++------ 2 files changed, 14 insertions(+), 11 deletions(-) diff --git a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/TensorPrimitives.netcore.cs b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/TensorPrimitives.netcore.cs index ed9bae090ebb4c..efdcfda4a6e7e2 100644 --- a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/TensorPrimitives.netcore.cs +++ b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/TensorPrimitives.netcore.cs @@ -1420,7 +1420,6 @@ private static int IndexOfMinMaxCore(ReadOnlySpan x) wher for (int i = 1; i < x.Length; i++) { float current = x[i]; - //float currentAbs = MathF.Abs(current); if (float.IsNaN(current)) { return i; @@ -1432,20 +1431,23 @@ private static int IndexOfMinMaxCore(ReadOnlySpan x) wher return curIn; } -#if NET8_0_OR_GREATER - private static int IndexOfFirstMatch(Vector512 mask) + private static int IndexOfFirstMatch(Vector128 mask) { return BitOperations.TrailingZeroCount(mask.ExtractMostSignificantBits()); } -#endif + private static int IndexOfFirstMatch(Vector256 mask) { return BitOperations.TrailingZeroCount(mask.ExtractMostSignificantBits()); } - private static int IndexOfFirstMatch(Vector128 mask) + +#if NET8_0_OR_GREATER + private static int IndexOfFirstMatch(Vector512 mask) { return BitOperations.TrailingZeroCount(mask.ExtractMostSignificantBits()); } +#endif + /// Performs an element-wise operation on and writes the results to . /// Specifies the operation to perform on each element loaded from . private static void InvokeSpanIntoSpan( diff --git a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/TensorPrimitives.netstandard.cs b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/TensorPrimitives.netstandard.cs index 4de2e2968aeb69..4dc8ffcf56d82c 100644 --- a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/TensorPrimitives.netstandard.cs +++ b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/TensorPrimitives.netstandard.cs @@ -311,6 +311,8 @@ private static float MinMaxCore(ReadOnlySpan x, TMinMaxO return result; } + private static readonly int[] s_0through7 = [0, 1, 2, 3, 4, 5, 6, 7]; + private static int IndexOfMinMaxCore(ReadOnlySpan x, TIndexOfMinMaxOperator op = default) where TIndexOfMinMaxOperator : struct, IIndexOfOperator { @@ -320,13 +322,13 @@ private static int IndexOfMinMaxCore(ReadOnlySpan // It treats +0 as greater than -0 as per the specification. int result; - int i; + int i = 0; if (Vector.IsHardwareAccelerated && Vector.Count <= 8 && x.Length >= Vector.Count) { ref float xRef = ref MemoryMarshal.GetReference(x); - Vector resultIndex = new Vector([0, 1, 2, 3, 4, 5, 6, 7], 0); + Vector resultIndex = new Vector(s_0through7); Vector curIndex = resultIndex; Vector increment = new Vector(Vector.Count); @@ -377,17 +379,16 @@ private static int IndexOfMinMaxCore(ReadOnlySpan // Scalar path used when either vectorization is not supported, the input is too small to vectorize, // or a NaN is encountered. Scalar: - float curResult = x[0]; - int curIn = 0; + float curResult = x[i]; + int curIn = i; if (float.IsNaN(curResult)) { return curIn; } - for (i = 1; i < x.Length; i++) + for (; i < x.Length; i++) { float current = x[i]; - //float currentAbs = MathF.Abs(current); if (float.IsNaN(current)) { return i; From d409ad6dd586fc6168534faf4379f3c4301b9f19 Mon Sep 17 00:00:00 2001 From: Michael Sharp Date: Fri, 20 Oct 2023 00:55:55 -0600 Subject: [PATCH 19/19] fixed spacing --- .../src/System/Numerics/Tensors/TensorPrimitives.netcore.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/TensorPrimitives.netcore.cs b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/TensorPrimitives.netcore.cs index efdcfda4a6e7e2..e82f25c889c522 100644 --- a/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/TensorPrimitives.netcore.cs +++ b/src/libraries/System.Numerics.Tensors/src/System/Numerics/Tensors/TensorPrimitives.netcore.cs @@ -8126,7 +8126,7 @@ public static void Invoke(ref Vector128 max, Vector128 current, re Vector128 greaterThanMask = Vector128.GreaterThan(max, current); Vector128 equalMask = Vector128.Equals(max, current); - if (equalMask.AsInt32() != Vector128.Zero) + if (equalMask.AsInt32() != Vector128.Zero) { Vector128 negativeMask = IsNegative(current); Vector128 lessThanMask = Vector128.LessThan(maxIndex, curIndex);