#pragma once #include #include #include #include #include "mma_sm89_fp16.hpp" #include "mma_traits_sm89_fp16.hpp" using namespace cute; template struct BytesToType {}; template<> struct BytesToType<4> { using Type = uint32_t; static_assert(sizeof(Type) == 4); }; template<> struct BytesToType<2> { using Type = uint16_t; static_assert(sizeof(Type) == 2); }; template struct gemm_traits { static constexpr int BLOCK_SIZE = 128; static constexpr int K = K_; static constexpr int BM = BM_; static constexpr int BN = BN_; static constexpr int BK = 128; static constexpr int TILES_PER_BLOCK = BLOCK_SIZE / BK; static constexpr int NUM_SFB_PER_STEP = BN / BLOCK_SIZE; static constexpr int NTiles = K / BK; static constexpr int KSF = K / BLOCK_SIZE; static constexpr int KStages = KStages_; static constexpr int WARP_ROW = WARP_ROW_; static constexpr int WARP_COL = WARP_COL_; static constexpr int NUM_WARPS = WARP_ROW * WARP_COL; static constexpr int NUM_THREADS = NUM_WARPS * 32; static constexpr int MMA_WARP_M = WARP_ROW * 16; static constexpr int MMA_WARP_N = WARP_COL * 8; static constexpr int MMA_WARP_K = 32; using accum_t = accum_t_; using out_t = out_t_; using SwizzleLayoutO = std::conditional_t< std::is_same_v, Swizzle<3, 3, 3>, Swizzle<2, 4, 3> >; using SwizzleLayoutAB = Swizzle<2, 4, 3>; using MMA_Atom_SM89 = std::conditional_t< std::is_same_v, MMA_Atom, MMA_Atom >; static constexpr int INPUT_ELEMS_PER_COPY = sizeof(uint128_t) / sizeof(float_e4m3_t); static constexpr int OUTPUT_ELEMS_PER_COPY = sizeof(uint128_t) / sizeof(out_t_); static constexpr int THREADS_PER_ROW = BK / INPUT_ELEMS_PER_COPY; using GMEMLayout = Layout< Shape , Int>, Stride, _1>>; using G2SCopyAtom = Copy_Atom, float_e4m3_t>; using G2STiledCopy = decltype( make_tiled_copy( G2SCopyAtom{}, GMEMLayout{}, Layout>>{} ) ); using S2RCopyAtomA = Copy_Atom; using S2RCopyAtomB = Copy_Atom; using SmemLayoutAtom = decltype(composition( Swizzle<2, 4, 3>{}, make_layout(make_shape(Int<8>{}, Int{}), make_stride(Int{}, Int<1>{})))); using SmemLayoutA = decltype( tile_to_shape(SmemLayoutAtom{}, make_shape(Int{}, Int{}, Int{})) ); using SmemLayoutB = decltype( tile_to_shape(SmemLayoutAtom{}, make_shape(Int{}, Int{}, Int{})) ); using MMATile = decltype( make_tiled_mma( MMA_Atom_SM89{}, Layout, Int, _1>>{}, Tile, Int, Int>{} ) ); static constexpr int ELEMS_PER_TILE = MMA_WARP_M * MMA_WARP_N; static constexpr int NUM_ELEMS_PER_WRITE = NUM_THREADS * sizeof(cute::uint128_t) / sizeof(out_t_); static constexpr int OUT_PIPE = NUM_ELEMS_PER_WRITE / ELEMS_PER_TILE; // using SmemLayoutC = Layout, Int>, Stride, Int<1>>>; using SmemLayoutC = decltype( make_layout( make_shape(Int{}, Int{}), make_stride(Int{}, Int<1>{}) ) ); static constexpr int THREADS_PER_ROW_WRITE = MMA_WARP_N * OUT_PIPE / OUTPUT_ELEMS_PER_COPY; using R2SCopyAtomC = Copy_Atom::Type>, out_t>; using S2GCopyAtomC = Copy_Atom, out_t>; using S2GCopyC = decltype(make_tiled_copy(S2GCopyAtomC{}, make_layout(make_shape(Int{}, Int{}), make_stride(Int{}, Int<1>{})), make_layout(make_shape(Int<1>{}, Int{})))); using G2SBiasCopyAtom = Copy_Atom, float>; using G2SBiasCopy = decltype(make_tiled_copy(G2SBiasCopyAtom{}, make_layout( make_shape(Int<1>{},Int{}), make_stride(Int{}, Int<1>{})), make_layout(make_shape(Int<1>{},Int<1>{}), make_stride(Int<1>{}, Int<1>{})))); using sfa_copy_vtype = float; static constexpr int SFA_ELEMS_PER_COPY = sizeof(sfa_copy_vtype)/sizeof(float); static constexpr int THREADS_SFA_COPY = BM * sizeof(float) / sizeof(sfa_copy_vtype); // using G2SSFACopyAtom = Copy_Atom, float>; static constexpr bool HasBias = HasBias_; using SmemLayoutBias = Layout, Int>, Stride, Int<1>>>; using SmemLayoutSFA = Layout, Int>, Stride, Int>>; using BiasThreadLayout = Layout, Shape, Int>>, Stride, Stride<_0, _8>>>; using SFAThreadLayout = Layout, Shape, Int>>, Stride, Stride<_16, _0>>>; static constexpr int SmemSize = cute::max(cute::cosize(SmemLayoutA{})+cute::cosize(SmemLayoutB{}), cute::cosize(SmemLayoutC{})*sizeof(out_t)) + cute::cosize(SmemLayoutBias{}) * sizeof(float) + cute::cosize(SmemLayoutSFA{})*sizeof(float); };