transpose_sse2.h 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367
  1. /*
  2. * Copyright (c) 2015 The WebM project authors. All Rights Reserved.
  3. *
  4. * Use of this source code is governed by a BSD-style license
  5. * that can be found in the LICENSE file in the root of the source
  6. * tree. An additional intellectual property rights grant can be found
  7. * in the file PATENTS. All contributing project authors may
  8. * be found in the AUTHORS file in the root of the source tree.
  9. */
  10. #ifndef VPX_VPX_DSP_X86_TRANSPOSE_SSE2_H_
  11. #define VPX_VPX_DSP_X86_TRANSPOSE_SSE2_H_
  12. #include <emmintrin.h> // SSE2
  13. #include "./vpx_config.h"
  14. static INLINE __m128i transpose_8bit_4x4(const __m128i *const in) {
  15. // Unpack 16 bit elements. Goes from:
  16. // in[0]: 00 01 02 03
  17. // in[1]: 10 11 12 13
  18. // in[2]: 20 21 22 23
  19. // in[3]: 30 31 32 33
  20. // to:
  21. // a0: 00 10 01 11 02 12 03 13
  22. // a1: 20 30 21 31 22 32 23 33
  23. const __m128i a0 = _mm_unpacklo_epi8(in[0], in[1]);
  24. const __m128i a1 = _mm_unpacklo_epi8(in[2], in[3]);
  25. // Unpack 32 bit elements resulting in:
  26. // 00 10 20 30 01 11 21 31 02 12 22 32 03 13 23 33
  27. return _mm_unpacklo_epi16(a0, a1);
  28. }
  29. static INLINE void transpose_8bit_8x8(const __m128i *const in,
  30. __m128i *const out) {
  31. // Unpack 8 bit elements. Goes from:
  32. // in[0]: 00 01 02 03 04 05 06 07
  33. // in[1]: 10 11 12 13 14 15 16 17
  34. // in[2]: 20 21 22 23 24 25 26 27
  35. // in[3]: 30 31 32 33 34 35 36 37
  36. // in[4]: 40 41 42 43 44 45 46 47
  37. // in[5]: 50 51 52 53 54 55 56 57
  38. // in[6]: 60 61 62 63 64 65 66 67
  39. // in[7]: 70 71 72 73 74 75 76 77
  40. // to:
  41. // a0: 00 10 01 11 02 12 03 13 04 14 05 15 06 16 07 17
  42. // a1: 20 30 21 31 22 32 23 33 24 34 25 35 26 36 27 37
  43. // a2: 40 50 41 51 42 52 43 53 44 54 45 55 46 56 47 57
  44. // a3: 60 70 61 71 62 72 63 73 64 74 65 75 66 76 67 77
  45. const __m128i a0 = _mm_unpacklo_epi8(in[0], in[1]);
  46. const __m128i a1 = _mm_unpacklo_epi8(in[2], in[3]);
  47. const __m128i a2 = _mm_unpacklo_epi8(in[4], in[5]);
  48. const __m128i a3 = _mm_unpacklo_epi8(in[6], in[7]);
  49. // Unpack 16 bit elements resulting in:
  50. // b0: 00 10 20 30 01 11 21 31 02 12 22 32 03 13 23 33
  51. // b1: 40 50 60 70 41 51 61 71 42 52 62 72 43 53 63 73
  52. // b2: 04 14 24 34 05 15 25 35 06 16 26 36 07 17 27 37
  53. // b3: 44 54 64 74 45 55 65 75 46 56 66 76 47 57 67 77
  54. const __m128i b0 = _mm_unpacklo_epi16(a0, a1);
  55. const __m128i b1 = _mm_unpackhi_epi16(a0, a1);
  56. const __m128i b2 = _mm_unpacklo_epi16(a2, a3);
  57. const __m128i b3 = _mm_unpackhi_epi16(a2, a3);
  58. // Unpack 32 bit elements resulting in:
  59. // c0: 00 10 20 30 40 50 60 70 01 11 21 31 41 51 61 71
  60. // c1: 02 12 22 32 42 52 62 72 03 13 23 33 43 53 63 73
  61. // c2: 04 14 24 34 44 54 64 74 05 15 25 35 45 55 65 75
  62. // c3: 06 16 26 36 46 56 66 76 07 17 27 37 47 57 67 77
  63. const __m128i c0 = _mm_unpacklo_epi32(b0, b2);
  64. const __m128i c1 = _mm_unpackhi_epi32(b0, b2);
  65. const __m128i c2 = _mm_unpacklo_epi32(b1, b3);
  66. const __m128i c3 = _mm_unpackhi_epi32(b1, b3);
  67. // Unpack 64 bit elements resulting in:
  68. // out[0]: 00 10 20 30 40 50 60 70
  69. // out[1]: 01 11 21 31 41 51 61 71
  70. // out[2]: 02 12 22 32 42 52 62 72
  71. // out[3]: 03 13 23 33 43 53 63 73
  72. // out[4]: 04 14 24 34 44 54 64 74
  73. // out[5]: 05 15 25 35 45 55 65 75
  74. // out[6]: 06 16 26 36 46 56 66 76
  75. // out[7]: 07 17 27 37 47 57 67 77
  76. out[0] = _mm_unpacklo_epi64(c0, c0);
  77. out[1] = _mm_unpackhi_epi64(c0, c0);
  78. out[2] = _mm_unpacklo_epi64(c1, c1);
  79. out[3] = _mm_unpackhi_epi64(c1, c1);
  80. out[4] = _mm_unpacklo_epi64(c2, c2);
  81. out[5] = _mm_unpackhi_epi64(c2, c2);
  82. out[6] = _mm_unpacklo_epi64(c3, c3);
  83. out[7] = _mm_unpackhi_epi64(c3, c3);
  84. }
  85. static INLINE void transpose_16bit_4x4(const __m128i *const in,
  86. __m128i *const out) {
  87. // Unpack 16 bit elements. Goes from:
  88. // in[0]: 00 01 02 03 XX XX XX XX
  89. // in[1]: 10 11 12 13 XX XX XX XX
  90. // in[2]: 20 21 22 23 XX XX XX XX
  91. // in[3]: 30 31 32 33 XX XX XX XX
  92. // to:
  93. // a0: 00 10 01 11 02 12 03 13
  94. // a1: 20 30 21 31 22 32 23 33
  95. const __m128i a0 = _mm_unpacklo_epi16(in[0], in[1]);
  96. const __m128i a1 = _mm_unpacklo_epi16(in[2], in[3]);
  97. // Unpack 32 bit elements resulting in:
  98. // out[0]: 00 10 20 30 01 11 21 31
  99. // out[1]: 02 12 22 32 03 13 23 33
  100. out[0] = _mm_unpacklo_epi32(a0, a1);
  101. out[1] = _mm_unpackhi_epi32(a0, a1);
  102. }
  103. static INLINE void transpose_16bit_4x8(const __m128i *const in,
  104. __m128i *const out) {
  105. // Unpack 16 bit elements. Goes from:
  106. // in[0]: 00 01 02 03 XX XX XX XX
  107. // in[1]: 10 11 12 13 XX XX XX XX
  108. // in[2]: 20 21 22 23 XX XX XX XX
  109. // in[3]: 30 31 32 33 XX XX XX XX
  110. // in[4]: 40 41 42 43 XX XX XX XX
  111. // in[5]: 50 51 52 53 XX XX XX XX
  112. // in[6]: 60 61 62 63 XX XX XX XX
  113. // in[7]: 70 71 72 73 XX XX XX XX
  114. // to:
  115. // a0: 00 10 01 11 02 12 03 13
  116. // a1: 20 30 21 31 22 32 23 33
  117. // a2: 40 50 41 51 42 52 43 53
  118. // a3: 60 70 61 71 62 72 63 73
  119. const __m128i a0 = _mm_unpacklo_epi16(in[0], in[1]);
  120. const __m128i a1 = _mm_unpacklo_epi16(in[2], in[3]);
  121. const __m128i a2 = _mm_unpacklo_epi16(in[4], in[5]);
  122. const __m128i a3 = _mm_unpacklo_epi16(in[6], in[7]);
  123. // Unpack 32 bit elements resulting in:
  124. // b0: 00 10 20 30 01 11 21 31
  125. // b1: 40 50 60 70 41 51 61 71
  126. // b2: 02 12 22 32 03 13 23 33
  127. // b3: 42 52 62 72 43 53 63 73
  128. const __m128i b0 = _mm_unpacklo_epi32(a0, a1);
  129. const __m128i b1 = _mm_unpacklo_epi32(a2, a3);
  130. const __m128i b2 = _mm_unpackhi_epi32(a0, a1);
  131. const __m128i b3 = _mm_unpackhi_epi32(a2, a3);
  132. // Unpack 64 bit elements resulting in:
  133. // out[0]: 00 10 20 30 40 50 60 70
  134. // out[1]: 01 11 21 31 41 51 61 71
  135. // out[2]: 02 12 22 32 42 52 62 72
  136. // out[3]: 03 13 23 33 43 53 63 73
  137. out[0] = _mm_unpacklo_epi64(b0, b1);
  138. out[1] = _mm_unpackhi_epi64(b0, b1);
  139. out[2] = _mm_unpacklo_epi64(b2, b3);
  140. out[3] = _mm_unpackhi_epi64(b2, b3);
  141. }
  142. static INLINE void transpose_16bit_8x8(const __m128i *const in,
  143. __m128i *const out) {
  144. // Unpack 16 bit elements. Goes from:
  145. // in[0]: 00 01 02 03 04 05 06 07
  146. // in[1]: 10 11 12 13 14 15 16 17
  147. // in[2]: 20 21 22 23 24 25 26 27
  148. // in[3]: 30 31 32 33 34 35 36 37
  149. // in[4]: 40 41 42 43 44 45 46 47
  150. // in[5]: 50 51 52 53 54 55 56 57
  151. // in[6]: 60 61 62 63 64 65 66 67
  152. // in[7]: 70 71 72 73 74 75 76 77
  153. // to:
  154. // a0: 00 10 01 11 02 12 03 13
  155. // a1: 20 30 21 31 22 32 23 33
  156. // a2: 40 50 41 51 42 52 43 53
  157. // a3: 60 70 61 71 62 72 63 73
  158. // a4: 04 14 05 15 06 16 07 17
  159. // a5: 24 34 25 35 26 36 27 37
  160. // a6: 44 54 45 55 46 56 47 57
  161. // a7: 64 74 65 75 66 76 67 77
  162. const __m128i a0 = _mm_unpacklo_epi16(in[0], in[1]);
  163. const __m128i a1 = _mm_unpacklo_epi16(in[2], in[3]);
  164. const __m128i a2 = _mm_unpacklo_epi16(in[4], in[5]);
  165. const __m128i a3 = _mm_unpacklo_epi16(in[6], in[7]);
  166. const __m128i a4 = _mm_unpackhi_epi16(in[0], in[1]);
  167. const __m128i a5 = _mm_unpackhi_epi16(in[2], in[3]);
  168. const __m128i a6 = _mm_unpackhi_epi16(in[4], in[5]);
  169. const __m128i a7 = _mm_unpackhi_epi16(in[6], in[7]);
  170. // Unpack 32 bit elements resulting in:
  171. // b0: 00 10 20 30 01 11 21 31
  172. // b1: 40 50 60 70 41 51 61 71
  173. // b2: 04 14 24 34 05 15 25 35
  174. // b3: 44 54 64 74 45 55 65 75
  175. // b4: 02 12 22 32 03 13 23 33
  176. // b5: 42 52 62 72 43 53 63 73
  177. // b6: 06 16 26 36 07 17 27 37
  178. // b7: 46 56 66 76 47 57 67 77
  179. const __m128i b0 = _mm_unpacklo_epi32(a0, a1);
  180. const __m128i b1 = _mm_unpacklo_epi32(a2, a3);
  181. const __m128i b2 = _mm_unpacklo_epi32(a4, a5);
  182. const __m128i b3 = _mm_unpacklo_epi32(a6, a7);
  183. const __m128i b4 = _mm_unpackhi_epi32(a0, a1);
  184. const __m128i b5 = _mm_unpackhi_epi32(a2, a3);
  185. const __m128i b6 = _mm_unpackhi_epi32(a4, a5);
  186. const __m128i b7 = _mm_unpackhi_epi32(a6, a7);
  187. // Unpack 64 bit elements resulting in:
  188. // out[0]: 00 10 20 30 40 50 60 70
  189. // out[1]: 01 11 21 31 41 51 61 71
  190. // out[2]: 02 12 22 32 42 52 62 72
  191. // out[3]: 03 13 23 33 43 53 63 73
  192. // out[4]: 04 14 24 34 44 54 64 74
  193. // out[5]: 05 15 25 35 45 55 65 75
  194. // out[6]: 06 16 26 36 46 56 66 76
  195. // out[7]: 07 17 27 37 47 57 67 77
  196. out[0] = _mm_unpacklo_epi64(b0, b1);
  197. out[1] = _mm_unpackhi_epi64(b0, b1);
  198. out[2] = _mm_unpacklo_epi64(b4, b5);
  199. out[3] = _mm_unpackhi_epi64(b4, b5);
  200. out[4] = _mm_unpacklo_epi64(b2, b3);
  201. out[5] = _mm_unpackhi_epi64(b2, b3);
  202. out[6] = _mm_unpacklo_epi64(b6, b7);
  203. out[7] = _mm_unpackhi_epi64(b6, b7);
  204. }
  205. // Transpose in-place
  206. static INLINE void transpose_16bit_16x16(__m128i *const left,
  207. __m128i *const right) {
  208. __m128i tbuf[8];
  209. transpose_16bit_8x8(left, left);
  210. transpose_16bit_8x8(right, tbuf);
  211. transpose_16bit_8x8(left + 8, right);
  212. transpose_16bit_8x8(right + 8, right + 8);
  213. left[8] = tbuf[0];
  214. left[9] = tbuf[1];
  215. left[10] = tbuf[2];
  216. left[11] = tbuf[3];
  217. left[12] = tbuf[4];
  218. left[13] = tbuf[5];
  219. left[14] = tbuf[6];
  220. left[15] = tbuf[7];
  221. }
  222. static INLINE void transpose_32bit_4x4(const __m128i *const in,
  223. __m128i *const out) {
  224. // Unpack 32 bit elements. Goes from:
  225. // in[0]: 00 01 02 03
  226. // in[1]: 10 11 12 13
  227. // in[2]: 20 21 22 23
  228. // in[3]: 30 31 32 33
  229. // to:
  230. // a0: 00 10 01 11
  231. // a1: 20 30 21 31
  232. // a2: 02 12 03 13
  233. // a3: 22 32 23 33
  234. const __m128i a0 = _mm_unpacklo_epi32(in[0], in[1]);
  235. const __m128i a1 = _mm_unpacklo_epi32(in[2], in[3]);
  236. const __m128i a2 = _mm_unpackhi_epi32(in[0], in[1]);
  237. const __m128i a3 = _mm_unpackhi_epi32(in[2], in[3]);
  238. // Unpack 64 bit elements resulting in:
  239. // out[0]: 00 10 20 30
  240. // out[1]: 01 11 21 31
  241. // out[2]: 02 12 22 32
  242. // out[3]: 03 13 23 33
  243. out[0] = _mm_unpacklo_epi64(a0, a1);
  244. out[1] = _mm_unpackhi_epi64(a0, a1);
  245. out[2] = _mm_unpacklo_epi64(a2, a3);
  246. out[3] = _mm_unpackhi_epi64(a2, a3);
  247. }
  248. static INLINE void transpose_32bit_4x4x2(const __m128i *const in,
  249. __m128i *const out) {
  250. // Unpack 32 bit elements. Goes from:
  251. // in[0]: 00 01 02 03
  252. // in[1]: 10 11 12 13
  253. // in[2]: 20 21 22 23
  254. // in[3]: 30 31 32 33
  255. // in[4]: 04 05 06 07
  256. // in[5]: 14 15 16 17
  257. // in[6]: 24 25 26 27
  258. // in[7]: 34 35 36 37
  259. // to:
  260. // a0: 00 10 01 11
  261. // a1: 20 30 21 31
  262. // a2: 02 12 03 13
  263. // a3: 22 32 23 33
  264. // a4: 04 14 05 15
  265. // a5: 24 34 25 35
  266. // a6: 06 16 07 17
  267. // a7: 26 36 27 37
  268. const __m128i a0 = _mm_unpacklo_epi32(in[0], in[1]);
  269. const __m128i a1 = _mm_unpacklo_epi32(in[2], in[3]);
  270. const __m128i a2 = _mm_unpackhi_epi32(in[0], in[1]);
  271. const __m128i a3 = _mm_unpackhi_epi32(in[2], in[3]);
  272. const __m128i a4 = _mm_unpacklo_epi32(in[4], in[5]);
  273. const __m128i a5 = _mm_unpacklo_epi32(in[6], in[7]);
  274. const __m128i a6 = _mm_unpackhi_epi32(in[4], in[5]);
  275. const __m128i a7 = _mm_unpackhi_epi32(in[6], in[7]);
  276. // Unpack 64 bit elements resulting in:
  277. // out[0]: 00 10 20 30
  278. // out[1]: 01 11 21 31
  279. // out[2]: 02 12 22 32
  280. // out[3]: 03 13 23 33
  281. // out[4]: 04 14 24 34
  282. // out[5]: 05 15 25 35
  283. // out[6]: 06 16 26 36
  284. // out[7]: 07 17 27 37
  285. out[0] = _mm_unpacklo_epi64(a0, a1);
  286. out[1] = _mm_unpackhi_epi64(a0, a1);
  287. out[2] = _mm_unpacklo_epi64(a2, a3);
  288. out[3] = _mm_unpackhi_epi64(a2, a3);
  289. out[4] = _mm_unpacklo_epi64(a4, a5);
  290. out[5] = _mm_unpackhi_epi64(a4, a5);
  291. out[6] = _mm_unpacklo_epi64(a6, a7);
  292. out[7] = _mm_unpackhi_epi64(a6, a7);
  293. }
  294. static INLINE void transpose_32bit_8x4(const __m128i *const in,
  295. __m128i *const out) {
  296. // Unpack 32 bit elements. Goes from:
  297. // in[0]: 00 01 02 03
  298. // in[1]: 04 05 06 07
  299. // in[2]: 10 11 12 13
  300. // in[3]: 14 15 16 17
  301. // in[4]: 20 21 22 23
  302. // in[5]: 24 25 26 27
  303. // in[6]: 30 31 32 33
  304. // in[7]: 34 35 36 37
  305. // to:
  306. // a0: 00 10 01 11
  307. // a1: 20 30 21 31
  308. // a2: 02 12 03 13
  309. // a3: 22 32 23 33
  310. // a4: 04 14 05 15
  311. // a5: 24 34 25 35
  312. // a6: 06 16 07 17
  313. // a7: 26 36 27 37
  314. const __m128i a0 = _mm_unpacklo_epi32(in[0], in[2]);
  315. const __m128i a1 = _mm_unpacklo_epi32(in[4], in[6]);
  316. const __m128i a2 = _mm_unpackhi_epi32(in[0], in[2]);
  317. const __m128i a3 = _mm_unpackhi_epi32(in[4], in[6]);
  318. const __m128i a4 = _mm_unpacklo_epi32(in[1], in[3]);
  319. const __m128i a5 = _mm_unpacklo_epi32(in[5], in[7]);
  320. const __m128i a6 = _mm_unpackhi_epi32(in[1], in[3]);
  321. const __m128i a7 = _mm_unpackhi_epi32(in[5], in[7]);
  322. // Unpack 64 bit elements resulting in:
  323. // out[0]: 00 10 20 30
  324. // out[1]: 01 11 21 31
  325. // out[2]: 02 12 22 32
  326. // out[3]: 03 13 23 33
  327. // out[4]: 04 14 24 34
  328. // out[5]: 05 15 25 35
  329. // out[6]: 06 16 26 36
  330. // out[7]: 07 17 27 37
  331. out[0] = _mm_unpacklo_epi64(a0, a1);
  332. out[1] = _mm_unpackhi_epi64(a0, a1);
  333. out[2] = _mm_unpacklo_epi64(a2, a3);
  334. out[3] = _mm_unpackhi_epi64(a2, a3);
  335. out[4] = _mm_unpacklo_epi64(a4, a5);
  336. out[5] = _mm_unpackhi_epi64(a4, a5);
  337. out[6] = _mm_unpacklo_epi64(a6, a7);
  338. out[7] = _mm_unpackhi_epi64(a6, a7);
  339. }
  340. #endif // VPX_VPX_DSP_X86_TRANSPOSE_SSE2_H_