Coverage for src/flag_gems/runtime/backend/_mthreads/ops/dropout.py: 0%

1import logging 

2 

3import torch 

4import triton 

5import triton.language as tl 

6 

7from flag_gems import runtime 

8from flag_gems.runtime import torch_device_fn 

9from flag_gems.utils.random_utils import ( 

10 philox_backend_seed_offset, 

11 uint_to_uniform_float, 

12) 

13 

14logger = logging.getLogger( 

15 f'flag_gems.runtime.backend._mthreads.ops.{__name__.split(".")[-1]}' 

16) 

17 

18 

19@triton.heuristics(runtime.get_heuristic_config("dropout")) 

20@triton.jit(do_not_specialize=["p", "philox_seed", "philox_offset"]) 

21def dropout_forward_kernel( 

22 X, 

23 Y, 

24 dropout_mask, 

25 N, 

26 p, 

27 philox_seed, 

28 philox_offset, 

29 BLOCK: tl.constexpr, 

30): 

31 UNROLL: tl.constexpr = 4 # philox generate 128 random bits at a time 

32 philox_seed = philox_seed.to(tl.int64) 

33 philox_offset = philox_offset.to(tl.int64) 

34 c0 = (philox_offset & 0xFFFFFFFF).to(tl.uint32) 

35 c1 = ((philox_offset >> 32) & 0xFFFFFFFF).to(tl.uint32) 

36 i4 = tl.program_id(0) * BLOCK + tl.arange(0, BLOCK) 

37 c0 += i4 

38 _O = c0 * 0 

39 r0, r1, r2, r3 = tl.philox(philox_seed, c0, c1, _O, _O) 

40 r0 = uint_to_uniform_float(r0) 

41 r1 = uint_to_uniform_float(r1) 

42 r2 = uint_to_uniform_float(r2) 

43 r3 = uint_to_uniform_float(r3) 

44 

45 mask0 = r0 > p 

46 mask1 = r1 > p 

47 mask2 = r2 > p 

48 mask3 = r3 > p 

49 p = 1.0 / (1.0 - p) 

50 

51 off_0 = ((tl.program_id(0) * BLOCK * UNROLL).to(tl.int64) + tl.arange(0, BLOCK)).to( 

52 tl.int64 

53 ) 

54 off_1 = off_0 + BLOCK 

55 off_2 = off_1 + BLOCK 

56 off_3 = off_2 + BLOCK 

57 

58 x0 = tl.load(X + off_0, mask=off_0 < N, other=0.0, eviction_policy="evict_first") 

59 x1 = tl.load(X + off_1, mask=off_1 < N, other=0.0, eviction_policy="evict_first") 

60 x2 = tl.load(X + off_2, mask=off_2 < N, other=0.0, eviction_policy="evict_first") 

61 x3 = tl.load(X + off_3, mask=off_3 < N, other=0.0, eviction_policy="evict_first") 

62 

63 y0 = x0 * p * mask0 # tl.where(mask0, x0 * p, 0.0) 

64 y1 = x1 * p * mask1 # tl.where(mask1, x1 * p, 0.0) 

65 y2 = x2 * p * mask2 # tl.where(mask2, x2 * p, 0.0) 

66 y3 = x3 * p * mask3 # tl.where(mask3, x3 * p, 0.0) 

67 

68 tl.store(dropout_mask + off_0, mask0, mask=off_0 < N, eviction_policy="evict_first") 

69 tl.store(dropout_mask + off_1, mask1, mask=off_1 < N, eviction_policy="evict_first") 

70 tl.store(dropout_mask + off_2, mask2, mask=off_2 < N, eviction_policy="evict_first") 

71 tl.store(dropout_mask + off_3, mask3, mask=off_3 < N, eviction_policy="evict_first") 

72 

73 tl.store(Y + off_0, y0, mask=off_0 < N, eviction_policy="evict_first") 

74 tl.store(Y + off_1, y1, mask=off_1 < N, eviction_policy="evict_first") 

75 tl.store(Y + off_2, y2, mask=off_2 < N, eviction_policy="evict_first") 

76 tl.store(Y + off_3, y3, mask=off_3 < N, eviction_policy="evict_first") 

77 

78 

79@triton.heuristics(runtime.get_heuristic_config("dropout")) 

80@triton.jit(do_not_specialize=["scale"]) 

81def dropout_backward_kernel( 

82 DY, 

83 DX, 

84 dropout_mask, 

85 N, 

86 scale, 

87 BLOCK: tl.constexpr, 

88): 

89 offset = ((tl.program_id(0) * BLOCK).to(tl.int64) + tl.arange(0, BLOCK)).to( 

90 tl.int64 

91 ) 

92 mask = offset < N 

93 m = tl.load( 

94 dropout_mask + offset, mask=mask, other=0, eviction_policy="evict_first" 

95 ) 

96 dy = tl.load(DY + offset, mask=mask, other=0, eviction_policy="evict_first") 

97 dx = dy * m * scale 

98 tl.store(DX + offset, dx, mask=mask, eviction_policy="evict_first") 

99 

100 

101UNROLL = 4 

102 

103 

104def dropout(input, p, train=True): 

105 logger.debug("GEMS_MTHREADS NATIVE DROPOUT FORWARD") 

106 if not train or p == 0: 

107 out = input.clone() 

108 mask = torch.ones_like(input, dtype=torch.bool) 

109 return out, mask 

110 if p == 1: 

111 out = torch.zeros_like(input) 

112 mask = torch.zeros_like(input, dtype=torch.bool) 

113 return out, mask 

114 assert p > 0.0 and p < 1.0, "p must be in (0, 1)" 

115 device = input.device 

116 # TODO: remove contiguous enforcement 

117 input = input.contiguous() 

118 out = torch.empty_like(input) 

119 mask = torch.empty_like(input, dtype=torch.bool) 

120 N = input.numel() 

121 grid_fn = lambda meta: (triton.cdiv(N, meta["BLOCK"] * UNROLL),) 

122 # (TODO) Using Triton autotuner makes kernel parameters opaque to the caller, 

123 # hence we cannot obtain the per thread offset as in Pytorch. 

124 increment = triton.cdiv(N, UNROLL) 

125 with torch_device_fn.device(device): 

126 philox_seed, philox_offset = philox_backend_seed_offset(increment) 

127 dropout_forward_kernel[grid_fn]( 

128 input, out, mask, N, p, philox_seed, philox_offset 

129 ) 

130 return out, mask 

131 

132 

133def dropout_backward(grad_output, mask, scale): 

134 logger.debug("GEMS NATIVE DROPOUT BACKWARD") 

135 grad_output = grad_output.contiguous() 

136 grad_input = torch.empty_like(grad_output) 

137 N = grad_output.numel() 

138 grid_fn = lambda meta: (triton.cdiv(N, meta["BLOCK"]),) 

139 with torch_device_fn.device(grad_output.device): 

140 dropout_backward_kernel[grid_fn](grad_output, grad_input, mask, N, scale) 

141 return grad_input